From 16afa019317ca8e32d8bddbfd750dc3f19c3d1f6 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:12:04 +0800 Subject: [PATCH 01/77] feat(workflow): add durable pull request delivery - Add deliver: pr for writable leaves, composite branches, fan-out items and loop passes. Persist post-agent delivery separately, serialize provider I/O outside the run lock, retain cancellation receipts and retry failed delivery without rerunning the agent or losing the pushed branch. - Record copy origins and resolve stacked targets through after dependencies, skipped/read-only steps and inherited workflow needs. Reuse open requests through existing GitHub, GitLab, TGit, GitCode and CNB authentication; retain the generic Git fallback and never invoke a provider merge API. - Inspect the exact delivered commit against the run base before PR creation, reject protected CI/agent/MCP/rules/hook paths, and preserve execution bounds across delivery. Reconcile limits reached by a report before provider I/O. - Validate and sanitize driver review URLs without exposing rejected values, report unknown changes fields by name, and publish ordered, deduplicated prs from validated changes only. Keep usage-report opt-out and done envelopes. - Document delivery and the unattended CI path in both languages, update help and dashboard state, and verify 1,311 related unit tests, 40 CLI E2E cases, focused provider/recovery/security tests, typecheck, lint and combined requirements/session/delivery integration against local fixture remotes. --- docs/dev/workflow-engine.md | 8 +- docs/guide/workflow.md | 51 ++- docs/guide/zh-CN/workflow.md | 51 ++- skill-data/core/references/commands.md | 2 +- .../e2e/workflow-conformance.test.ts | 4 +- .../e2e/workflow-pr-delivery-1090.test.ts | 102 ++++++ src/__tests__/workflow-conformance.test.ts | 4 +- src/__tests__/workflow-delivery.test.ts | 293 ++++++++++++++++++ src/__tests__/workflow-docs-examples.test.ts | 2 + src/__tests__/workflow-pr-providers.test.ts | 97 ++++++ src/__tests__/workflow-records.test.ts | 2 +- src/dashboard/client.ts | 2 +- src/dashboard/locales.ts | 1 + src/dashboard/workflow-runs.ts | 4 +- src/index.ts | 2 +- src/providers/cnb/cnb-cli.ts | 8 +- src/providers/cnb/index.ts | 4 + src/providers/cnb/workflow-pr.ts | 41 +++ src/providers/gitcode/gitcode-api.ts | 2 + src/providers/gitcode/index.ts | 4 + src/providers/gitcode/workflow-pr.ts | 37 +++ src/providers/github/gh-cli.ts | 1 + src/providers/github/index.ts | 9 + src/providers/github/workflow-pr.ts | 48 +++ src/providers/gitlab/gitlab-api.ts | 2 + src/providers/gitlab/index.ts | 10 + src/providers/gitlab/workflow-pr.ts | 42 +++ src/providers/tgit/index.ts | 4 + src/providers/tgit/workflow-pr.ts | 32 ++ src/providers/types.ts | 6 + src/workflow/conformance.ts | 30 +- src/workflow/delivery.ts | 192 ++++++++++++ src/workflow/engine.ts | 114 ++++++- src/workflow/fan-out.ts | 2 +- src/workflow/local-driver.ts | 4 + src/workflow/parse.ts | 11 +- src/workflow/pr-url.ts | 16 + src/workflow/records.ts | 12 + src/workflow/run-controls.ts | 10 +- src/workflow/step-state.ts | 22 +- src/workflow/types.ts | 7 +- src/workflow/workspace.ts | 18 +- 42 files changed, 1272 insertions(+), 41 deletions(-) create mode 100644 src/__tests__/e2e/workflow-pr-delivery-1090.test.ts create mode 100644 src/__tests__/workflow-delivery.test.ts create mode 100644 src/__tests__/workflow-pr-providers.test.ts create mode 100644 src/providers/cnb/workflow-pr.ts create mode 100644 src/providers/gitcode/workflow-pr.ts create mode 100644 src/providers/github/workflow-pr.ts create mode 100644 src/providers/gitlab/workflow-pr.ts create mode 100644 src/providers/tgit/workflow-pr.ts create mode 100644 src/workflow/delivery.ts create mode 100644 src/workflow/pr-url.ts diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 10301b488..7383fc075 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,7 +11,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. +- **A workspace is a git contract on the lease; the driver executes it, and PR delivery inspects its committed result.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push is `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, and starts fresh when resume is not available. OpenCode has no resume flag, so it starts fresh. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. @@ -39,4 +39,10 @@ Each rule below spans more than one file. The rule names the code that owns it, ## Tests +PR delivery (`delivery.ts`) runs after the agent's successful report or composite settlement, in a durable `delivering` state. It is neither stopped nor settled, counts toward fan-out concurrency, and can be cancelled. A provider failure becomes `delivery_failed`; retry preserves the branch and returns to delivery, without another agent attempt. `next` reconciles first, then drains deliveries outside the run lock under a separate reclaimable delivery lock; provider lookup makes a restart after creation idempotent. Drivers poll a transient `wait.delivery`. No delivery code calls a merge API. + +Shared delivery state readers/writers: `rememberCopy` records `copies[].origin` once from the actual workspace start/first merge, before the driver updates `copies[].commit` in `report`. Workspace planning reads commit/owner; delivery walks origin to the nearest validated delivered branch, falling back to `run.base.ref`. `changes.pr_url` is validated outside the lock and accepted only by the report transaction that owns the lease. The engine stamps `prRecordedAt`; drivers cannot set it. Step schemas, loop passes and archives retain both fields. End records read only these changes fields, order by revision, sanitize again and deduplicate; output URLs are excluded and reporting opt-out applies unchanged. `done` gains no fields. + +Delivery inspects the exact reported commit against the run base using `git diff --no-renames`, with shared `AGENT_PATHS` plus CI/hook paths. It never trusts a reported diff or commit list. All providers use their existing authentication, with optional open-request lookup and repository/branch validation; arbitrary HTTPS review services remain allowed without sending them provider credentials. Generic Git retains transport-only behavior and reports the preserved branch on failure. + Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass the conformance suite in `src/workflow/conformance.ts`. If a driver can see a new behavior, add a scenario there. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c97b3cba5..11b884ffc 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -155,7 +155,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. +`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` and `changes.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. The run is stored in this scope's TeamAI data directory. The command exits when it finishes. A local run that ends `failed` or `cancelled` exits with status 1. No workflow process keeps running. @@ -198,8 +198,57 @@ steps: command: node -e "require('fs').writeFileSync('joined.txt', 'J')" ``` +## Deliver a PR + +Set `deliver: pr` on a writable step, or on a `workflow` step whose children share its `workspace: new` branch. TeamAI pushes the branch and opens a review request through the product repository's Git provider. It never merges it. A `for_each` delivers one PR per item. A looped `new` step gets a new `/p` branch and PR each pass; earlier PRs stay open. + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +The PR targets the nearest delivered branch along the branch's recorded origin; `for_each.after` comes before other `needs`, and the first child inherits the enclosing workflow's dependencies. Skipped or read-only dependencies are passed over. If no earlier branch delivered, the target is the run's original branch. Start from a named branch: a detached checkout has no PR target. An open PR for the same head and target is reused. + +Agent completion is followed by a durable `delivering` state. Dependents wait for it. Git and provider operations happen outside the run lock. A PR creation failure reports `delivery_failed` and names the preserved branch. Fix provider authentication, run `teamai workflow retry `, then `teamai workflow resume `: this retries delivery on that branch without rerunning the agent. Push failures remain `push_failed` and retain the worktree. A completed composite may already have removed its worktree; its committed, pushed branch remains. Generic Git hosts support transport only: create a PR yourself and report its URL, or use a provider that supports PR creation. No interactive login starts during delivery. + +Before opening a PR, TeamAI checks the delivered commit against the run's base commit. Changes to CI, hooks, agent instructions, rules, MCP and equivalent tool configuration are rejected, with the paths named. The shared agent configuration paths are the same ones restored before workspace execution. This check includes changes inherited from earlier branches and deletions/renames. + +A remote driver may include `changes.pr_url` alongside `branch`, `base`, `from`, `to` and `commits`. Recognized provider metadata is checked against the expected repository and branches. Any HTTPS host/path is allowed for private review services. URLs containing credentials, invalid/non-HTTPS URLs, and URLs longer than 512 characters are dropped; query strings and fragments are stripped. A dropped URL does not reject the report: `dropped` lists `{ field: "changes.pr_url", reason: "…" }` without the value. Unknown nested fields are listed in `ignored` as `changes.`. Drivers must still obey a lease's workspace and commit contract. + +### Start from CI + +Store the workflow above in the team repo and configure a local prompt tool there. A normal GitHub Actions job is the v1 trigger path. The token must read the team repo, push branches to the product repo, and create PRs there. Configure `TEAM_REPO` and `WORKFLOW_TOKEN` in the repository settings. + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` answers optional approve requests and optional choose defaults only. A required gate fails with `hitl_required` in an unattended job. To start from a failing branch, check that branch out by name; the default `pull_request` checkout is a detached merge commit. TeamAI does not install triggers or wait for CI/merges. + ## Run records +The end record includes `prs`, the validated `changes.pr_url` links in first-appearance order, once per URL across items and loop passes. URLs from step outputs are never published. `prs` follows the same usage-report opt-out as the rest of the record and is public when the team repo is public. Older records without `prs` remain readable. + With [usage reporting](reporting.md) on, every run publishes a privacy-preserving record to the team's `teamai-reports` branch: once when it starts and once when it ends. With it off, nothing is published and the envelopes carry no `record`. Records live in `workflows/runs/.json`, keyed by run id, so a retried publication updates the same record instead of adding a copy. A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 8f8bd05cf..d059afcab 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -155,7 +155,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 +`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>` 和 `changes.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 运行状态保存在当前 scope 的 TeamAI 数据目录里。命令结束后进程就退出。本地运行以 `failed` 或 `cancelled` 结束时，退出码是 1。不会留下常驻的工作流进程。 @@ -198,8 +198,57 @@ steps: command: node -e "require('fs').writeFileSync('joined.txt', 'J')" ``` +## 交付 PR + +在可写步骤上设置 `deliver: pr`，或在子步骤共用其 `workspace: new` 分支的 `workflow` 步骤上设置它。TeamAI 推送分支，通过产品仓库的 Git provider 创建供人审查的 PR，从不合并。`for_each` 每项交付一个 PR。循环中的 `new` 步骤每个 pass 使用新的 `/p` 分支和 PR，之前的 PR 保持打开。 + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +PR 的目标沿分支记录的来源，追溯到最近已经交付的分支。`for_each.after` 优先于其他 `needs`，第一个子步骤继承外层工作流的依赖；跳过或只读的依赖会继续向前追溯。之前没有交付分支时，目标为运行开始时的分支。因此请从有名称的分支开始；detached checkout 没有 PR 目标。同一源分支和目标分支已有打开的 PR 时会复用。 + +agent 完成后，步骤进入持久的 `delivering` 状态，后续依赖等待交付结束。Git 和 provider 操作不占用运行锁。创建 PR 失败时以 `delivery_failed` 结束，消息写明保留的分支。修好认证后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，只重试这条分支的交付，不重跑 agent。推送失败仍为 `push_failed`，并保留 worktree。组合工作流结束时可能已移除 worktree，但提交和远程分支仍在。通用 Git 主机只支持传输：请自行创建 PR 并上报 URL，或使用支持创建 PR 的 provider。交付不会启动交互登录。 + +创建 PR 前，TeamAI 将交付提交与运行的基准提交比较。CI、hooks、agent 指令、rules、MCP 和同类工具配置的修改会被拒绝，并列出路径。共享 agent 配置路径与工作区执行前恢复的路径相同。检查也覆盖从前序分支继承的修改，以及删除和重命名。 + +远程驱动可在 `changes` 的 `branch`、`base`、`from`、`to`、`commits` 之外上报 `pr_url`。provider 能提供元数据时，会校验仓库和源、目标分支。私有审查服务可使用任意 HTTPS 主机和路径。带凭据、无效、非 HTTPS 或超过 512 字符的 URL 被丢弃；query 和 fragment 会移除。丢弃 URL 不会拒绝 report，`dropped` 以 `{ field: "changes.pr_url", reason: "…" }` 列出原因，不包含值。未知嵌套字段以 `changes.<字段>` 列入 `ignored`。驱动仍须遵守租约的工作区与提交契约。 + +### 从 CI 启动 + +将上面的工作流放进团队仓库，并在其中配置本地提示词工具。普通 GitHub Actions 作业是 v1 的触发方式。token 需要能读取团队仓库、向产品仓库推送分支和创建 PR。在仓库设置里配置 `TEAM_REPO` 和 `WORKFLOW_TOKEN`。 + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` 只回答可选 approve 和有默认值的可选 choose。必需的 gate 在无人值守作业中以 `hitl_required` 失败。从失败分支启动时，按分支名检出；默认的 `pull_request` checkout 是 detached merge commit。TeamAI 不安装触发器，也不等待 CI 或合并事件。 + ## 运行记录 +结束记录包含 `prs`：通过校验的 `changes.pr_url` 按首次出现顺序排列，跨 item 和循环 pass 去重。步骤 outputs 中的 URL 不会发布。`prs` 遵循相同的 usage-report 退出设置；团队仓库公开时它也公开。没有 `prs` 的旧记录仍可读取。 + 开启[使用统计上报](reporting.md)时，每次运行都会向团队的 `teamai-reports` 分支发布一条保护隐私的记录：开始时一次，结束时一次。关闭后不发布任何记录，返回中也不带 `record`。记录存放在 `workflows/runs/.json`，以 run id 为键，重试的发布会更新同一条记录而不是产生副本。 记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数，以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index b0277fa9b..fa18054b1 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -220,7 +220,7 @@ Generated: do not edit by hand. Regenerate with ## workflow -- `teamai workflow` — Run a validated workflow from workflows/.yaml +- `teamai workflow` — Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them. - `teamai workflow start ` — Create a run. With --driver local, execute command and prompt steps in this process. - `--driver ` — Run the workflow in this process. Only local is available. - `--tools ` — Override local prompt tools for this run, as comma-separated name=tool pairs diff --git a/src/__tests__/e2e/workflow-conformance.test.ts b/src/__tests__/e2e/workflow-conformance.test.ts index 8bed1d885..f1e1e1d5e 100644 --- a/src/__tests__/e2e/workflow-conformance.test.ts +++ b/src/__tests__/e2e/workflow-conformance.test.ts @@ -73,7 +73,7 @@ describe('workflow protocol conformance through the CLI', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -81,6 +81,7 @@ describe('workflow protocol conformance through the CLI', () => { driver: { ...driver, ...(operation.hitl !== undefined ? { can: { hitl: operation.hitl } } : {}) }, ...(operation.running ? { running: operation.running.map(attemptOf) } : {}), ...(operation.canTakeLease !== undefined ? { canTakeLease: operation.canTakeLease } : {}), + ...(operation.workspace ? { driver: { ...driver, can: { workspace: ['read-only', 'new', 'continue'] } }, base: { ref: 'main', commit: 'a'.repeat(40) } } : {}), }); expect(value).toMatchObject(operation.expect); if (operation.saveAs === 'busy') { @@ -111,6 +112,7 @@ describe('workflow protocol conformance through the CLI', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, + ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), }); expect(value).toMatchObject(operation.expect); } diff --git a/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts new file mode 100644 index 000000000..59535baf7 --- /dev/null +++ b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts @@ -0,0 +1,102 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { trackDetachedProcesses } from '../helpers/detached-processes.js'; +import type { RunState } from '../../workflow/types.js'; + +const cli = fileURLToPath(new URL('../../../dist/index.js', import.meta.url)); + +describe('workflow PR delivery CLI (#1090)', () => { + it('runs the documented CI path, creates/reuses a provider PR, retains generic-git work, and refuses required unattended approvals', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-pr-cli-'))); + const home = path.join(root, 'home'); const product = path.join(root, 'product'); const team = path.join(root, 'team'); const bin = path.join(root, 'bin'); + for (const dir of [home, product, team, bin]) fs.mkdirSync(dir); + const detached = trackDetachedProcesses(root); + const env: NodeJS.ProcessEnv = { ...process.env, HOME: home, USERPROFILE: home, PATH: `${bin}${path.delimiter}${process.env.PATH}`, FORCE_COLOR: '0', + GIT_CONFIG_GLOBAL: path.join(home, '.gitconfig'), GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', + GIT_AUTHOR_NAME: 'Fixture', GIT_AUTHOR_EMAIL: 'fixture@example.invalid', GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + NODE_OPTIONS: [process.env.NODE_OPTIONS, detached.nodeOptions].filter(Boolean).join(' '), PR_FIXTURE: path.join(root, 'pr.json') }; + delete env.CLAUDE_CONFIG_DIR; delete env.CODEX_HOME; delete env.GITHUB_TOKEN; delete env.GH_TOKEN; + const run = (command: string, args: string[], cwd = product) => spawnSync(command, args, { cwd, env, encoding: 'utf8', timeout: 30_000 }); + const git = (args: string[], cwd = product) => { const result = run('git', args, cwd); expect(result.status, result.stderr).toBe(0); return result.stdout.trim(); }; + const call = (args: string[]) => run(process.execPath, [cli, ...args]); + const envelope = (stdout: string) => JSON.parse(stdout.trim().split('\n').filter((line) => line.startsWith('{')).at(-1)!); + const productUrl = 'https://github.com/fixture/app.git'; const teamUrl = 'https://team.example.invalid/fixture/team.git'; + const bare = path.join(root, 'product.git'); + fs.writeFileSync(path.join(bin, 'gh'), `#!${process.execPath} +const fs = require('fs'); const args = process.argv.slice(2); const file = process.env.PR_FIXTURE; +if (args[0] === '--version') { console.log('gh version 2.80.0'); process.exit(0); } +if (args[0] === 'api') { + const saved = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null; + console.log(JSON.stringify(args[1].includes('?') ? saved ? [saved] : [] : saved)); process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'create') { + const val = key => args[args.indexOf(key) + 1]; + if (val('-R') !== 'fixture/app' || val('-B') !== 'main') process.exit(7); + const pr = { state: 'open', html_url: 'https://github.com/fixture/app/pull/41', head: { ref: val('-H'), repo: { full_name: 'fixture/app' } }, base: { ref: val('-B'), repo: { full_name: 'fixture/app' } } }; + fs.writeFileSync(file, JSON.stringify(pr)); fs.appendFileSync(file + '.creates', 'create\\n'); console.log(pr.html_url); process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'list') { console.log('[]'); process.exit(0); } +console.error('Unexpected provider operation: ' + args[0]); process.exit(9); +`, { mode: 0o755 }); + try { + git(['init', '-q', '-b', 'main']); fs.writeFileSync(path.join(product, 'README.md'), '# Product\n'); git(['add', '.']); git(['commit', '-qm', 'base']); + git(['init', '-q', '--bare', '-b', 'main', bare]); git(['remote', 'add', 'origin', productUrl]); + git(['config', '--global', `url.${bare}.insteadOf`, productUrl]); git(['push', '-qu', 'origin', 'main']); + git(['init', '-q', '-b', 'main'], team); fs.mkdirSync(path.join(team, 'workflows')); + fs.writeFileSync(path.join(team, 'teamai.yaml'), `team: fixture\nrepo: ${teamUrl}\nprovider: git\nusageReport: false\n`); + const yaml = `schemaVersion: v1alpha1 +name: ship-issue +description: Deliver an issue change for review. +inputs: { issue: number } +steps: + - id: approve + approve: { message: Start the fixture?, required: false } + - id: work + needs: [approve] + workspace: new + deliver: pr + command: node -e "require('fs').writeFileSync('issue.txt', process.env.ISSUE)" + env: + ISSUE: "{{ inputs.issue }}" +`; + fs.writeFileSync(path.join(team, 'workflows', 'ship-issue.yaml'), yaml); + fs.writeFileSync(path.join(team, 'workflows', 'required.yaml'), yaml.replace('name: ship-issue', 'name: required').replace('required: false', 'required: true')); + git(['add', '.'], team); git(['commit', '-qm', 'team workflows'], team); + git(['config', '--global', `url.${team}.insteadOf`, teamUrl]); + const initialized = call(['init', teamUrl, '--scope', 'workspace', '--agent', 'claude', '--force']); expect(initialized.status, `${initialized.stdout}\n${initialized.stderr}`).toBe(0); + const pulled = call(['pull']); expect(pulled.status, pulled.stderr).toBe(0); await detached.waitForExit(); + const started = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":42}']); + expect(started.status, `${started.stdout}\n${started.stderr}`).toBe(0); + const done = envelope(started.stdout).data.done; expect(done.outcome).toBe('succeeded'); + const runFile = fs.readdirSync(root, { recursive: true, encoding: 'utf8' }).find((file) => file.endsWith(path.join('workflows', 'runs', `${done.run}.json`))); + expect(runFile).toBeDefined(); + const dataHome = path.dirname(path.dirname(path.dirname(path.join(root, runFile!)))); + const read = (id: string) => JSON.parse(fs.readFileSync(path.join(dataHome, 'workflows', 'runs', `${id}.json`), 'utf8')) as RunState; + const state = read(done.run); const work = state.steps.find((step) => step.id === 'work'); + expect(work).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/41' } }); + const branch = `teamai/${done.run}/work/p1`; + expect(git(['--git-dir', bare, 'show', `${branch}:issue.txt`])).toBe('42'); expect(fs.existsSync(path.join(product, 'issue.txt'))).toBe(false); + // Re-enter durable post-agent delivery as after a crash: the open PR is reused, with no new agent work or create. + if (work?.status !== 'succeeded') throw new Error('missing successful work'); + const persisted = state as unknown as { status: string; steps: Array> }; + persisted.status = 'running'; const target = persisted.steps.find((step) => step.id === 'work')!; target.status = 'delivering'; delete target.outcome; + delete (target.changes as Record).pr_url; delete (target.changes as Record).prRecordedAt; + fs.writeFileSync(path.join(dataHome, 'workflows', 'runs', `${done.run}.json`), JSON.stringify(persisted)); + const resumed = call(['workflow', 'resume', done.run]); expect(resumed.status, resumed.stderr).toBe(0); + expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); + git(['remote', 'set-url', 'origin', bare]); + const generic = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":43}']); + expect(generic.status).toBe(1); const genericDone = envelope(generic.stdout).data.done; + const failed = read(genericDone.run).steps.find((step) => step.id === 'work'); const failedBranch = `teamai/${genericDone.run}/work/p1`; + expect(failed).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(failedBranch) } }); + expect(git(['--git-dir', bare, 'show', `${failedBranch}:issue.txt`])).toBe('43'); + const required = call(['workflow', 'start', 'required', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":44}']); + expect(required.status).toBe(1); expect(read(envelope(required.stdout).data.done.run).steps[0]).toMatchObject({ failure: { code: 'hitl_required' } }); + expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); + } finally { await detached.waitForExit(); fs.rmSync(root, { recursive: true, force: true }); } + }); +}); diff --git a/src/__tests__/workflow-conformance.test.ts b/src/__tests__/workflow-conformance.test.ts index 95121e4bc..41e18975c 100644 --- a/src/__tests__/workflow-conformance.test.ts +++ b/src/__tests__/workflow-conformance.test.ts @@ -64,7 +64,7 @@ describe('workflow protocol conformance in process', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -72,6 +72,7 @@ describe('workflow protocol conformance in process', () => { driver: { ...driver, ...(operation.hitl !== undefined ? { can: { hitl: operation.hitl } } : {}) }, ...(operation.running ? { running: operation.running.map(attemptOf) } : {}), ...(operation.canTakeLease !== undefined ? { canTakeLease: operation.canTakeLease } : {}), + ...(operation.workspace ? { driver: { ...driver, can: { workspace: ['read-only', 'new', 'continue'] } }, base: { ref: 'main', commit: 'a'.repeat(40) } } : {}), }); expect(envelope).toMatchObject(operation.expect); if (operation.saveAs === 'busy') { @@ -100,6 +101,7 @@ describe('workflow protocol conformance in process', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, + ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), }); expect(envelope).toMatchObject(operation.expect); } diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts new file mode 100644 index 000000000..7ab3666af --- /dev/null +++ b/src/__tests__/workflow-delivery.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fc from 'fast-check'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { parseWorkflow } from '../workflow/parse.js'; +import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; +import { sanitizePrUrl } from '../workflow/pr-url.js'; +import { deliveryTarget, protectedDeliveryPaths, validateReportedPr } from '../workflow/delivery.js'; +import { syncWorkflowDefinitions } from '../workflow/cache.js'; +import { readRun, writeRun } from '../workflow/store.js'; +import { runLocalDriver } from '../workflow/local-driver.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import { buildEndRecord } from '../workflow/records.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import { settled, stopped } from '../workflow/step-state.js'; +import type { LocalConfig } from '../types.js'; +import type { RunState } from '../workflow/types.js'; + +const dirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function git(cwd: string, args: string[]): string { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim(); +} + +async function fixture(steps: string, inputs?: unknown): Promise<{ config: LocalConfig; cwd: string; runId: string; root: string }> { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-delivery-')); + dirs.push(root); + const cwd = path.join(root, 'product'); + const knowledge = path.join(root, 'team'); + fs.mkdirSync(cwd); + fs.mkdirSync(path.join(knowledge, 'workflows'), { recursive: true }); + git(root, ['init', '-q', '--bare', 'remote.git']); + git(cwd, ['init', '-q', '-b', 'main']); + git(cwd, ['config', 'user.name', 'Test']); + git(cwd, ['config', 'user.email', 'test@example.com']); + fs.writeFileSync(path.join(cwd, '.gitignore'), '.teamai/\n'); + fs.writeFileSync(path.join(cwd, 'base.txt'), 'base\n'); + git(cwd, ['add', '-A']); git(cwd, ['commit', '-qm', 'base']); + git(cwd, ['remote', 'add', 'origin', 'https://github.com/fixture/app.git']); + git(cwd, ['config', `url.${path.join(root, 'remote.git')}.insteadOf`, 'https://github.com/fixture/app.git']); + git(cwd, ['push', '-qu', 'origin', 'main']); + const config: LocalConfig = { repo: { localPath: knowledge, remote: 'local', kind: 'self' }, scope: 'project', + username: 'tester', usageReportEnabled: false, additionalRoles: [], projectRoot: cwd, dataHome: path.join(root, 'data'), workflows: { tools: {}, max_parallel: 1 } }; + fs.writeFileSync(path.join(knowledge, 'workflows', 'ship.yaml'), `schemaVersion: v1alpha1\nname: ship\ndescription: Deliver work\n${inputs ? 'inputs:\n packages: [{ name: string, id: string, after: [string] }]\n' : ''}steps:\n${steps}`); + await syncWorkflowDefinitions(config); + const started = await startRun(config, 'ship', inputs ?? {}, { checkout: cwd, base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) } }); + if ('error' in started) throw new Error(started.error.message); + return { config, cwd, root, runId: String(started.data.run) }; +} + +async function state(config: LocalConfig, runId: string): Promise { + const run = await readRun(config, runId); + if (typeof run === 'string') throw new Error(run); + return run; +} + +const command = `node -e "require('fs').writeFileSync('change.txt', 'change')"`; +const driver = { kind: 'local', machine: 'fixture', workspace: 'fixture' }; + +function provider() { + const lookup = vi.spyOn(GitHubProvider.prototype, 'findOpenPullRequest').mockResolvedValue(undefined); + const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest').mockResolvedValue(true); + let number = 0; + const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockImplementation(async () => `https://github.com/fixture/app/pull/${++number}`); + return { lookup, validate, create }; +} + +describe('workflow PR delivery contract', () => { + it('parses delivery on a writable step and rejects unsupported delivery', () => { + const yaml = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: write\n command: node write.js\n workspace: new\n deliver: pr\n'; + const parsed = parseWorkflow('ship.yaml', yaml); + expect(parsed.ok && parsed.definition.steps[0]).toMatchObject({ deliver: 'pr' }); + expect(parseWorkflow('ship.yaml', yaml.replace('deliver: pr', 'deliver: merge')).ok).toBe(false); + }); + + it('keeps the URL for validation and names unread nested fields without values', () => { + const parsed = parseReport({ attempt: 'a1', revision: 1, outcome: 'success', + driver: { kind: 'remote', machine: 'host', workspace: 'work' }, + changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', private: 'secret' } }); + expect(parsed).toMatchObject({ report: { changes: { pr_url: 'https://review.example/1' } }, ignored: ['changes.private'] }); + expect(JSON.stringify(parsed)).not.toContain('secret'); + }); + + it.each([ + ['https://github.com/acme/app/pull/101', 'https://github.com/acme/app/pull/101'], + ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview'], + ['https://review.acme.internal/changes/4821?session=abc#c3', 'https://review.acme.internal/changes/4821'], + ['https://user:token@github.com/acme/app/pull/1', undefined], + ['http://github.com/acme/app/pull/1', undefined], ['not a url', undefined], + [`https://example.com/${'a'.repeat(512)}`, undefined], + ])('sanitizes %s', (value, expected) => expect(sanitizePrUrl(value).url).toBe(expected)); + + it('never preserves credentials, queries or fragments and is idempotent and bounded', () => { + fc.assert(fc.property(fc.webUrl({ validSchemes: ['https', 'http'] }), (value) => { + const result = sanitizePrUrl(value); + if (!result.url) return; + const url = new URL(result.url); + expect(url.protocol).toBe('https:'); + expect([url.username, url.password, url.search, url.hash]).toEqual(['', '', '', '']); + expect(result.url.length).toBeLessThanOrEqual(512); + expect(sanitizePrUrl(result.url)).toEqual(result); + }), { numRuns: 300 }); + }); +}); + +describe('durable PR delivery', () => { + it('pushes a branch and opens a PR against the recorded base without merging', async () => { + const api = provider(); + const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/1' } }); + expect(api.create).toHaveBeenCalledWith(expect.objectContaining({ repo: 'fixture/app', source: `teamai/${runId}/write/p1`, target: 'main' })); + expect(git(root, ['--git-dir=remote.git', 'show', `refs/heads/teamai/${runId}/write/p1:change.txt`])).toBe('change'); + expect(git(cwd, ['show', 'main:base.txt'])).toBe('base'); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('keeps pushed work after missing credentials and retries delivery without rerunning the agent', async () => { + const api = provider(); + api.lookup.mockRejectedValue(new Error('Set GITHUB_TOKEN before resuming.')); + const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); + let run = await state(config, runId); + const branch = `teamai/${runId}/write/p1`; + expect(run.steps[0]).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(branch) } }); + expect(git(root, ['--git-dir=remote.git', 'show', `${branch}:change.txt`])).toBe('change'); + const attempts = run.settledCount; + api.lookup.mockResolvedValue('https://github.com/fixture/app/pull/88'); + expect(await retryStep(config, runId, 'write')).toMatchObject({ data: { reopened: true } }); + run = await state(config, runId); + expect(run.steps[0]?.status).toBe('delivering'); + expect(settled(run.steps[0])).toBe(false); + expect(stopped(run.steps[0]!)).toBe(false); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + run = await state(config, runId); + expect(run.settledCount).toBe(attempts); + expect(api.create).not.toHaveBeenCalled(); + expect(run.steps[0]).toMatchObject({ changes: { branch, pr_url: 'https://github.com/fixture/app/pull/88' } }); + }); + + it('checks protected changes against the run base before any provider request', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/evil.yml','bad')"\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/evil.yml') } }); + expect(api.create).not.toHaveBeenCalled(); expect(api.lookup).not.toHaveBeenCalled(); + expect(protectedDeliveryPaths(['src/a.ts', '.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit'])).toEqual(['.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit']); + }); + + it('delivers a continued composite branch once and follows inherited needs', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: change\n needs: [first]\n workspace: new\n deliver: pr\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').writeFileSync('second.txt','second')"\n - id: review\n needs: [write]\n command: cat second.txt\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create).toHaveBeenCalledTimes(2); + expect(api.create.mock.calls[1]?.[0]).toMatchObject({ source: `teamai/${runId}/change/p1`, target: `teamai/${runId}/first/p1` }); + expect((await state(config, runId)).steps.find((step) => step.id === 'change')).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/2' } }); + }); + + it('delivers each after-item against its actual origin and publishes PRs in item order', async () => { + const api = provider(); + const { config, cwd, root, runId } = await fixture(` - id: packages\n for_each:\n items: inputs.packages\n workspace: new\n deliver: pr\n command: node -e "require('fs').writeFileSync(process.env.PACKAGE+'.txt',process.env.PACKAGE)"\n env:\n PACKAGE: "{{ item.name }}"\n`, { packages: [{ name: 'one', id: 'one', after: [] }, { name: 'two', id: 'two', after: ['one'] }, { name: 'three', id: 'three', after: ['two'] }] }); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls.map(([opts]) => opts.target)).toEqual(['main', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`]); + expect(git(root, ['--git-dir=remote.git', 'diff', '--name-only', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`])).toBe('two.txt'); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); + }); + + it('accepts sanitized remote links, drops unsafe/mismatched ones without exposing their values', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); + const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'], base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) } }); + const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; + if (!lease) throw new Error(JSON.stringify(next)); + const changes = { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: 'https://user:token@github.com/fixture/app/pull/1' }; + const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes }); + expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); + expect(JSON.stringify([accepted, await state(config, runId)])).not.toContain('token'); + expect(api.validate).not.toHaveBeenCalled(); + }); + + it.each([ + { url: 'https://review.example/changes/1?session=secret#note', valid: undefined, expected: 'https://review.example/changes/1', reason: undefined }, + { url: 'https://github.com/fixture/app/pull/1', valid: false, expected: undefined, reason: 'repository or branches do not match the lease' }, + ])('records only a validated driver link ($url)', async ({ url, valid, expected, reason }) => { + const api = provider(); api.validate.mockResolvedValue(valid); + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); + const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'] }); + const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; + if (!lease) throw new Error('missing lease'); + const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes: { + branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: url, + } }); + expect(accepted).toMatchObject({ data: { accepted: true, ...(reason ? { dropped: [{ field: 'changes.pr_url', reason }] } : {}) } }); + const run = await state(config, runId); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(expected ? [expected] : undefined); + expect(JSON.stringify(run)).not.toContain('session='); + }); + + it('can cancel a persisted delivery without treating it as settled success', async () => { + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + const run = await state(config, runId); + run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; + await writeRun(config, run); + expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + }); + + it('walks undelivered origin branches and deduplicates URLs from passes and archives', async () => { + const { config, runId } = await fixture(` - id: write\n command: echo test\n`); + const run = await state(config, runId); + const changes = { branch: 'first', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', prRecordedAt: 2 }; + run.copies = { second: { owner: driver, origin: 'first' }, third: { owner: driver, origin: 'second' } }; + run.steps = [{ id: 'write', status: 'succeeded', outcome: 'success', settledAt: 4, changes: { ...changes, branch: 'third', prRecordedAt: 4 }, passes: [{ pass: 1, outcome: 'success', changes }] }]; + run.archive = [{ step: 'older', pass: 1, outcome: 'success', changes: { ...changes, pr_url: 'https://review.example/2', prRecordedAt: 3 } }]; + expect(deliveryTarget(run, 'third')).toBe('first'); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://review.example/1', 'https://review.example/2']); + if (run.steps[0]?.status === 'succeeded') run.steps[0].outputs = { pr_url: 'https://untrusted.example/no' }; + expect(buildEndRecord(run, driver, 'tester').prs).not.toContain('https://untrusted.example/no'); + }); + + it('opens a separate PR per new-workspace loop pass and retains all earlier URLs', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls.map(([opts]) => opts.source)).toEqual([1, 2, 3].map((pass) => `teamai/${runId}/write/p${pass}`)); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); + }); + + it('reuses one PR when a loop continues the same branch', async () => { + const api = provider(); + api.lookup.mockImplementation(async () => api.create.mock.calls.length ? 'https://github.com/fixture/app/pull/1' : undefined); + const { config, cwd, runId } = await fixture(` - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').appendFileSync('change.txt','pass\\n')"\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create).toHaveBeenCalledTimes(1); expect(api.lookup).toHaveBeenCalledTimes(3); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('passes over a skipped first need to the earlier delivered branch', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: other\n needs: []\n workspace: new\n command: node -e "require('fs').writeFileSync('other.txt','other')"\n deliver: pr\n - id: skipped\n needs: [first]\n if: steps.first.outcome == 'failed'\n command: echo skipped\n - id: last\n needs: [skipped, other]\n workspace: new\n command: node -e "require('fs').writeFileSync('last.txt','last')"\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls[2]?.[0]).toMatchObject({ source: `teamai/${runId}/last/p1`, target: `teamai/${runId}/first/p1` }); + }); + + it('rejects inherited protected changes even when the delivering step did not edit those paths', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: earlier\n workspace: new\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/ci.yml','bad')"\n - id: later\n needs: [earlier]\n workspace: new\n command: ${command}\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/ci.yml') } }); + expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); + }); + + it('allows cancellation during provider I/O and retains the resulting PR receipt without reviving the step', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + api.create.mockImplementation(async () => { + expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); + return 'https://github.com/fixture/app/pull/99'; + }); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'cancelled', changes: { pr_url: 'https://github.com/fixture/app/pull/99' } }); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/99']); + }); + + it('keeps an execution deadline across delivery and makes no provider call once it expired', async () => { + const api = provider(); + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + const run = await state(config, runId); + run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, deliveryDeadline: Date.now() - 1, + changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; + await writeRun(config, run); + expect(await leaseNext(config, runId, driver)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { code: 'timeout' } }); + expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); + }); + + it.each(['https://oauth2:fixture-token@github.com/fixture/app.git', 'ssh://git@github.com:22/fixture/app.git', 'git@github.com:fixture/app.git'])('recognizes hosting metadata through %s without passing transport credentials', async (remote) => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: echo test\n`); + git(cwd, ['remote', 'set-url', 'origin', remote]); + expect(await validateReportedPr(await state(config, runId), { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ url: 'https://github.com/fixture/app/pull/1' }); + expect(api.validate).toHaveBeenCalledWith('https://github.com/fixture/app/pull/1', expect.objectContaining({ repo: 'fixture/app' })); + expect(JSON.stringify(api.validate.mock.calls)).not.toContain('fixture-token'); + }); +}); diff --git a/src/__tests__/workflow-docs-examples.test.ts b/src/__tests__/workflow-docs-examples.test.ts index a09d97f00..904999920 100644 --- a/src/__tests__/workflow-docs-examples.test.ts +++ b/src/__tests__/workflow-docs-examples.test.ts @@ -68,6 +68,8 @@ function startsOf(page: string): { starts: Start[]; unread: string[] } { const unread = [...text.matchAll(/--inputs[ =]['"{]/g)] .map((match) => lineOf(match.index)) .filter((line) => !starts.some((start) => start.line === line)) + // Actions expressions become JSON only inside the job; the documented CLI path has its own real-CLI E2E. + .filter((line) => !text.split('\n')[line - 1]?.includes('${{')) .map((line) => `${page}:${line}: --inputs is not part of a teamai workflow start line`); return { starts, unread }; } diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts new file mode 100644 index 000000000..311b48532 --- /dev/null +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as github from '../providers/github/gh-cli.js'; +import * as gitcode from '../providers/gitcode/gitcode-api.js'; +import * as tgit from '../providers/tgit/rest-auth.js'; +import * as cnb from '../providers/cnb/cnb-cli.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import { GitLabProvider } from '../providers/gitlab/index.js'; +import { GitCodeProvider } from '../providers/gitcode/index.js'; +import { TGitProvider } from '../providers/tgit/index.js'; +import { CNBProvider } from '../providers/cnb/index.js'; +import type { PrCreateOptions } from '../providers/types.js'; + +const opts: PrCreateOptions = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', title: 'Change', spawnTimeoutMs: 1000 }; +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); + +describe('workflow provider PR reuse and validation', () => { + it('reuses the exact GitHub repository/head/base and never accepts a fork with the same branch name', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + const pull = { state: 'open', html_url: 'https://github.com/acme/app/pull/7', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: 'main', repo: { full_name: opts.repo } } }; + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ ...pull, head: { ref: opts.source, repo: { full_name: 'fork/app' } }, html_url: 'https://github.com/acme/app/pull/6' }, pull]))); + vi.stubGlobal('fetch', fetcher); + const provider = new GitHubProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe(pull.html_url); + const url = new URL(fetcher.mock.calls[0]![0] as string); + expect(url.searchParams.get('head')).toBe(`acme:${opts.source}`); + expect(url.searchParams.get('base')).toBe('main'); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify(pull))); + expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(true); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...pull, base: { ...pull.base, ref: 'other' } }))); + expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(false); + expect(await provider.validatePullRequest('https://github.com/other/app/pull/7', opts)).toBe(false); + const calls = fetcher.mock.calls.length; + expect(await provider.validatePullRequest('https://private-review.example/1', opts)).toBeUndefined(); + expect(fetcher).toHaveBeenCalledTimes(calls); + }); + + it('uses an already authenticated gh without invoking login', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(true); + const cli = vi.spyOn(github, 'ghExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([{ state: 'open', html_url: 'https://github.com/acme/app/pull/8', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }]) }); + expect(await new GitHubProvider().findOpenPullRequest(opts)).toBe('https://github.com/acme/app/pull/8'); + expect(cli.mock.calls[0]?.[0][0]).toBe('api'); + expect(cli.mock.calls.flatMap(([args]) => args)).not.toContain('login'); + }); + + it('fails noninteractively without GitHub credentials, without echoing response bodies', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); + vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); + await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub authentication unavailable'); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('private-token-in-body', { status: 403 }))); + await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub PR lookup failed (403). Check token and repository access.'); + }); + + it('validates GitLab project and branches at the configured instance, including a subpath', async () => { + vi.stubEnv('GITLAB_TOKEN', 'test-token'); vi.stubEnv('GITLAB_URL', 'https://gitlab.example/team'); + const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 5, target_project_id: 5, web_url: 'https://gitlab.example/team/acme/app/-/merge_requests/9' }; + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([mr]))); + vi.stubGlobal('fetch', fetcher); + const provider = new GitLabProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe(mr.web_url); + expect(String(fetcher.mock.calls[0]?.[0])).toContain('/team/api/v4/projects/acme%2Fapp/merge_requests?'); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify(mr))); + expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(true); + expect(await provider.validatePullRequest('https://gitlab.example/team/other/app/-/merge_requests/9', opts)).toBe(false); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...mr, source_project_id: 99 }))); + expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(false); + }); + + it('reuses TGit requests through its existing REST authentication', async () => { + const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 4, target_project_id: 4, web_url: 'https://git.woa.com/acme/app/merge_requests/3' }; + const fetcher = vi.spyOn(tgit, 'tgitFetch').mockResolvedValue(new Response(JSON.stringify([mr]))); + expect(await new TGitProvider().findOpenPullRequest(opts)).toBe(mr.web_url); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify([mr]))); + expect(await new TGitProvider().validatePullRequest(mr.web_url, opts)).toBe(true); + }); + + it('paginates GitCode open requests before deciding to create a new one', async () => { + vi.spyOn(gitcode, 'getGitCodeToken').mockReturnValue('test-token'); + const pull = { state: 'open', html_url: 'https://gitcode.com/acme/app/pull/2', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }; + const fetcher = vi.fn().mockResolvedValueOnce(new Response(JSON.stringify(Array.from({ length: 100 }, () => ({ ...pull, head: { ...pull.head, ref: 'other' } }))))) + .mockResolvedValueOnce(new Response(JSON.stringify([pull]))); + vi.stubGlobal('fetch', fetcher); + expect(await new GitCodeProvider().findOpenPullRequest(opts)).toBe(pull.html_url); + expect(String(fetcher.mock.calls[1]?.[0])).toContain('page=2'); + }); + + it('uses the CNB CLI schema and normalizes refs/heads branch names', async () => { + const pull = { state: 'open', number: '12', head: { ref: `refs/heads/${opts.source}`, repo: { path: opts.repo } }, base: { ref: 'refs/heads/main', repo: { path: opts.repo } } }; + const cli = vi.spyOn(cnb, 'cnbExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([pull]) }); + const provider = new CNBProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe('https://cnb.cool/acme/app/-/pulls/12'); + expect(cli.mock.calls[0]?.[0]).toContain('list-pulls'); + cli.mockReturnValueOnce({ status: 0, stderr: '', stdout: JSON.stringify(pull) }); + expect(await provider.validatePullRequest('https://cnb.cool/acme/app/-/pulls/12', opts)).toBe(true); + }); +}); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index b8c79b439..c9b4edc6b 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -203,7 +203,7 @@ describe('#1083 workflow run records', () => { it('publishes the failure code of every step failure teamai writes, and only codes it can name', () => { // Codes a `code: '…'` literal in src/workflow writes that never become a run's failure. - const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace'] satisfies Array; + const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'bad_delivery'] satisfies Array; const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'busy']; const source = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'workflow'); const written = new Set(fs.readdirSync(source).filter((file) => file.endsWith('.ts')) diff --git a/src/dashboard/client.ts b/src/dashboard/client.ts index aa87c7c44..6743e5a65 100644 --- a/src/dashboard/client.ts +++ b/src/dashboard/client.ts @@ -188,7 +188,7 @@ async function saveSelection() { if(!dist||!distDraft||!preview)return; const ge // records the team published. Rows carry only run-record fields, and a HITL // answer posts the pass the page showed so a replaced request is refused. const runStatusNames = { running:'Running', succeeded:'Succeeded', failed:'Failed', cancelled:'Cancelled' }; -const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; +const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', delivering:'Delivering PR', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; function runStatus(status) { return ''+label(runStatusNames[status]||stepStatusNames[status]||status)+''; } // A running run's duration is filled by tickRunDurations, so a poll that changes nothing leaves the HTML, and the buttons in it, alone. function runDurationHtml(r) { return r.durationMs==null&&r.status==='running'?'':e(elapsed(r.durationMs ?? NaN)); } diff --git a/src/dashboard/locales.ts b/src/dashboard/locales.ts index 86027f9b1..6191a7d06 100644 --- a/src/dashboard/locales.ts +++ b/src/dashboard/locales.ts @@ -264,6 +264,7 @@ export const dashboardMessages: Record = { "Cancelled": "已取消", "Pending": "待运行", "Leased": "已租用", + "Delivering PR": "正在交付 PR", "Skipped": "已跳过", "Waiting for an answer": "等待回答", "Waiting": "等待中", diff --git a/src/dashboard/workflow-runs.ts b/src/dashboard/workflow-runs.ts index e61da3b16..c61b423d7 100644 --- a/src/dashboard/workflow-runs.ts +++ b/src/dashboard/workflow-runs.ts @@ -90,7 +90,7 @@ interface RunRow { /** A run on this machine, as its run file stands now. */ export type LocalRunRow = RunRow & { source: 'local'; - /** The leased steps: a driver may work on several at once. */ + /** Agent leases and deliveries currently in progress. */ current: string[]; /** The steps whose HITL request waits for an answer. */ waiting: string[]; @@ -167,7 +167,7 @@ function localRow(run: RunState, records: Map): LocalRunRow return { ...rowOf(localRecord(run)), source: 'local', - current: run.steps.filter((state) => state.status === 'leased').map((state) => state.id), + current: run.steps.filter((state) => state.status === 'leased' || state.status === 'delivering').map((state) => state.id), waiting: run.steps.filter((state) => state.status === 'waiting').map((state) => state.id), ...(record ? { record } : {}), }; diff --git a/src/index.ts b/src/index.ts index dc614de0c..373e6fd6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -789,7 +789,7 @@ envCmd const workflowCmd = program .command('workflow') - .description('Run a validated workflow from workflows/.yaml'); + .description('Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them.'); workflowCmd .command('start ') diff --git a/src/providers/cnb/cnb-cli.ts b/src/providers/cnb/cnb-cli.ts index dc85b8d00..43e24785c 100644 --- a/src/providers/cnb/cnb-cli.ts +++ b/src/providers/cnb/cnb-cli.ts @@ -43,7 +43,7 @@ export const CNB_HOST = process.env.TEAMAI_CNB_HOST?.trim() || 'cnb.cool'; /** Run a `cnb` subcommand. Returns { stdout, stderr, status }. */ export function cnbExec( args: string[], - options?: { inheritStdio?: boolean; cwd?: string }, + options?: { inheritStdio?: boolean; cwd?: string; timeoutMs?: number }, ): { stdout: string; stderr: string; status: number } { // Resolve the executable first, then launch it through cross-spawn: on // Windows the npm-installed CLI is only a `.cmd` shim, which neither a bare @@ -58,7 +58,7 @@ export function cnbExec( log.debug(`cnb exec: ${cnbPath} ${args.join(' ')}`); if (options?.inheritStdio) { - const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd }); + const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd, ...(options.timeoutMs ? { timeout: options.timeoutMs } : {}) }); return { stdout: '', stderr: '', status: r.status ?? 1 }; } const r = crossSpawn.sync(cnbPath, args, { @@ -66,6 +66,7 @@ export function cnbExec( encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024, cwd: options?.cwd, + ...(options?.timeoutMs ? { timeout: options.timeoutMs } : {}), }); return { stdout: (r.stdout ?? '').toString().trim(), @@ -339,6 +340,7 @@ export interface CnbPullCreateOptions { title: string; description?: string; cwd?: string; + spawnTimeoutMs?: number; } /** @@ -357,7 +359,7 @@ export function cnbPullCreate(opts: CnbPullCreateOptions): string { ]; if (opts.description) args.push('--body', opts.description); - const r = cnbExec(args, { cwd: opts.cwd }); + const r = cnbExec(args, { cwd: opts.cwd, ...(opts.spawnTimeoutMs ? { timeoutMs: opts.spawnTimeoutMs } : {}) }); if (r.status !== 0) { throw new Error(`cnb post-pull failed: ${r.stderr || r.stdout}`); } diff --git a/src/providers/cnb/index.ts b/src/providers/cnb/index.ts index b3a3a02a9..2eb62aa2e 100644 --- a/src/providers/cnb/index.ts +++ b/src/providers/cnb/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, RepoInfo, PrCreateOptions } from '../types.js'; +import { findCnbPull, validateCnbPull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { cnbParseRepoInput, @@ -19,6 +20,8 @@ import { * the TGit provider's "thin class over a platform CLI" shape. */ export class CNBProvider implements GitProvider { + findOpenPullRequest(opts: PrCreateOptions): Promise { return findCnbPull(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateCnbPull(url, opts); } readonly name = 'cnb'; parseRepoInput(input: string): RepoInfo { @@ -72,6 +75,7 @@ export class CNBProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/cnb/workflow-pr.ts b/src/providers/cnb/workflow-pr.ts new file mode 100644 index 000000000..23fde2e45 --- /dev/null +++ b/src/providers/cnb/workflow-pr.ts @@ -0,0 +1,41 @@ +import { parse } from 'yaml'; +import type { PrCreateOptions } from '../types.js'; +import { CNB_HOST, cnbExec } from './cnb-cli.js'; + +interface Repository { path?: string; web_url?: string } +interface Pull { number: string; state: string; head: { ref: string; repo?: Repository }; base: { ref: string; repo?: Repository } } +function matches(pull: Pull, opts: PrCreateOptions): boolean { + const ref = (value: string): string => value.replace(/^refs\/heads\//, ''); + const repoMatches = (repo: Repository | undefined): boolean => { + if (repo?.web_url) { + const url = new URL(repo.web_url); + return url.host === CNB_HOST && url.pathname.replace(/^\/|\/$/g, '') === opts.repo; + } + return !repo?.path || repo.path === opts.repo || repo.path === opts.repo.split('/').at(-1); + }; + return pull.state === 'open' && ref(pull.head.ref) === opts.source && ref(pull.base.ref) === opts.target + && repoMatches(pull.head.repo) && repoMatches(pull.base.repo); +} +function request(command: string, args: string[], opts: PrCreateOptions): unknown { + const response = cnbExec(['pulls', command, '--repo', opts.repo, ...args], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); + if (response.status !== 0) throw new Error('CNB PR lookup failed. Check cnb authentication or CNB_TOKEN and repository access.'); + try { return parse(response.stdout) as unknown; } + catch { throw new Error('CNB PR lookup returned invalid data. Inspect cnb before retrying.'); } +} +export async function findCnbPull(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const pulls = request('list-pulls', ['--state', 'open', '--base-ref', opts.target, '--page-size', '100', '--page', String(page)], opts) as Pull[]; + if (!Array.isArray(pulls)) throw new Error('CNB PR lookup returned no request list. Inspect cnb before retrying.'); + const found = pulls.find((pull) => matches(pull, opts)); + if (found) return `https://${CNB_HOST}/${opts.repo}/-/pulls/${found.number}`; + if (pulls.length < 100) return undefined; + } + throw new Error('Too many open CNB requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateCnbPull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.host !== CNB_HOST) return undefined; + const prefix = `/${opts.repo}/-/pulls/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(request('get-pull', ['--number', url.pathname.slice(prefix.length).replace(/\/$/, '')], opts) as Pull, opts); +} diff --git a/src/providers/gitcode/gitcode-api.ts b/src/providers/gitcode/gitcode-api.ts index b4ea6da60..f51c155d0 100644 --- a/src/providers/gitcode/gitcode-api.ts +++ b/src/providers/gitcode/gitcode-api.ts @@ -274,6 +274,7 @@ export interface GitCodePullCreateOptions { description?: string; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } /** @@ -296,6 +297,7 @@ export async function gitcodePullCreate(opts: GitCodePullCreateOptions): Promise headers: authHeaders(token), body: JSON.stringify(body), redirect: 'manual', + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), }); if (!resp.ok) { const errBody = await resp.text().catch(() => ''); diff --git a/src/providers/gitcode/index.ts b/src/providers/gitcode/index.ts index 88ec983d2..e710844a8 100644 --- a/src/providers/gitcode/index.ts +++ b/src/providers/gitcode/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import { findGitCodePull, validateGitCodePull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { gitcodeIsAuthenticated, @@ -25,6 +26,8 @@ import { askQuestion, isInteractive } from '../../utils/prompt.js'; * that distinguish it from the GitLab provider it is structurally modelled on. */ export class GitCodeProvider implements GitProvider { + findOpenPullRequest(opts: PrCreateOptions): Promise { return findGitCodePull(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateGitCodePull(url, opts); } readonly name = 'gitcode'; parseRepoInput(input: string): RepoInfo { @@ -111,6 +114,7 @@ export class GitCodeProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/gitcode/workflow-pr.ts b/src/providers/gitcode/workflow-pr.ts new file mode 100644 index 000000000..675337d82 --- /dev/null +++ b/src/providers/gitcode/workflow-pr.ts @@ -0,0 +1,37 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitCodeToken, gitcodeApiBase } from './gitcode-api.js'; +import { GITCODE_HOST } from './repo-url.js'; + +interface Pull { html_url: string; state: string; head: { ref: string; repo?: { full_name?: string } }; base: { ref: string; repo?: { full_name?: string } } } +function matches(pull: Pull, opts: PrCreateOptions): boolean { + return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target + && (!pull.head.repo?.full_name || pull.head.repo.full_name === opts.repo) + && (!pull.base.repo?.full_name || pull.base.repo.full_name === opts.repo); +} +async function request(route: string, opts: PrCreateOptions): Promise { + const token = getGitCodeToken(); + if (!token) throw new Error('GitCode authentication unavailable. Set GITCODE_TOKEN before resuming.'); + const response = await fetch(`${gitcodeApiBase()}/repos/${opts.repo}/pulls${route}`, { + headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' }, redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitCode PR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('gitcode review lookup returned invalid data.'); }); +} +export async function findGitCodePull(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const query = new URLSearchParams({ state: 'open', base: opts.target, per_page: '100', page: String(page) }); + const pulls = await request(`?${query}`, opts) as Pull[]; + const found = pulls.find((pull) => matches(pull, opts)); + if (found) return found.html_url; + if (pulls.length < 100) return undefined; + } + throw new Error('Too many open GitCode requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateGitCodePull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.hostname !== GITCODE_HOST) return undefined; + const prefix = `/${opts.repo}/pull/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as Pull, opts); +} diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index 9aae33eb6..0f3eb8fe7 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -556,6 +556,7 @@ async function ghPrCreateViaApi(opts: GhPrCreateOptions): Promise { { method: 'POST', headers: authHeaders, + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), body: JSON.stringify({ title: opts.title, body: opts.description ?? opts.title, diff --git a/src/providers/github/index.ts b/src/providers/github/index.ts index 9d7af0342..b24c3fbf7 100644 --- a/src/providers/github/index.ts +++ b/src/providers/github/index.ts @@ -12,6 +12,7 @@ import { } from './gh-cli.js'; import { ghListOrgRepos } from './gh-org.js'; import { parseGitHubRepoInput } from './repo-url.js'; +import { findGitHubPull, validateGitHubPull } from './workflow-pr.js'; export class GitHubProvider implements GitProvider { readonly name = 'github'; @@ -64,6 +65,14 @@ export class GitHubProvider implements GitProvider { }); } + findOpenPullRequest(opts: PrCreateOptions): Promise { + return findGitHubPull(opts); + } + + validatePullRequest(url: string, opts: PrCreateOptions): Promise { + return validateGitHubPull(url, opts); + } + getDefaultEmailDomain(): string | null { return null; } diff --git a/src/providers/github/workflow-pr.ts b/src/providers/github/workflow-pr.ts new file mode 100644 index 000000000..dde389cd4 --- /dev/null +++ b/src/providers/github/workflow-pr.ts @@ -0,0 +1,48 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitHubToken, ghExec, isGhInstalled } from './gh-cli.js'; + +interface Pull { + html_url: string; + state: string; + head: { ref: string; repo: { full_name: string } | null }; + base: { ref: string; repo: { full_name: string } }; +} + +async function request(path: string, opts: PrCreateOptions): Promise { + if (isGhInstalled()) { + const result = ghExec(['api', path], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); + if (result.status !== 0) throw new Error('GitHub PR lookup failed. Check gh authentication and repository access.'); + try { return JSON.parse(result.stdout) as unknown; } + catch { throw new Error('GitHub PR lookup returned invalid data. Inspect gh before retrying.'); } + } + const token = getGitHubToken(); + if (!token) throw new Error('GitHub authentication unavailable. Set GITHUB_TOKEN or authenticate gh before resuming.'); + const response = await fetch(`https://api.github.com/${path}`, { + headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, + redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitHub PR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('GitHub PR lookup returned invalid data.'); }); +} + +function matches(pull: Pull, opts: PrCreateOptions): boolean { + return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target + && pull.head.repo?.full_name.toLowerCase() === opts.repo.toLowerCase() + && pull.base.repo.full_name.toLowerCase() === opts.repo.toLowerCase(); +} + +export async function findGitHubPull(opts: PrCreateOptions): Promise { + const query = new URLSearchParams({ state: 'open', head: `${opts.repo.split('/')[0]}:${opts.source}`, base: opts.target, per_page: '100' }); + const pulls = await request(`repos/${opts.repo}/pulls?${query}`, opts) as Pull[]; + return pulls.find((pull) => matches(pull, opts))?.html_url; +} + +export async function validateGitHubPull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + // A remote driver may use a separate review service. Validate recognized provider URLs only. + if (url.hostname !== 'github.com') return undefined; + const match = /^\/([^/]+\/[^/]+)\/pull\/(\d+)\/?$/.exec(url.pathname); + if (!match || match[1]!.toLowerCase() !== opts.repo.toLowerCase()) return false; + return matches(await request(`repos/${opts.repo}/pulls/${match[2]}`, opts) as Pull, opts); +} diff --git a/src/providers/gitlab/gitlab-api.ts b/src/providers/gitlab/gitlab-api.ts index 9409bf0d5..a2a426cc4 100644 --- a/src/providers/gitlab/gitlab-api.ts +++ b/src/providers/gitlab/gitlab-api.ts @@ -301,6 +301,7 @@ export interface GitLabMrCreateOptions { reviewers?: string[]; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } interface GitLabUser { @@ -355,6 +356,7 @@ export async function gitlabMrCreate(opts: GitLabMrCreateOptions): Promise ''); diff --git a/src/providers/gitlab/index.ts b/src/providers/gitlab/index.ts index 506a810a5..5d05e8f90 100644 --- a/src/providers/gitlab/index.ts +++ b/src/providers/gitlab/index.ts @@ -13,6 +13,7 @@ import { gitlabListOrgRepos } from './org.js'; import { fetchGitLabMR } from './mr-fetch.js'; import { parseGitLabRepoInput } from './repo-url.js'; import type { MRData } from '../../types.js'; +import { findGitLabMergeRequest, validateGitLabMergeRequest } from './workflow-pr.js'; /** * GitLab provider (supports self-hosted / enterprise GitLab instances). @@ -72,6 +73,7 @@ export class GitLabProvider implements GitProvider { description: opts.description, reviewers: opts.reviewers, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } @@ -79,6 +81,14 @@ export class GitLabProvider implements GitProvider { return fetchGitLabMR(url); } + findOpenPullRequest(opts: PrCreateOptions): Promise { + return findGitLabMergeRequest(opts); + } + + validatePullRequest(url: string, opts: PrCreateOptions): Promise { + return validateGitLabMergeRequest(url, opts); + } + async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gitlabListOrgRepos(org, opts); } diff --git a/src/providers/gitlab/workflow-pr.ts b/src/providers/gitlab/workflow-pr.ts new file mode 100644 index 000000000..cf8c1ed9a --- /dev/null +++ b/src/providers/gitlab/workflow-pr.ts @@ -0,0 +1,42 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitLabToken, gitlabApiBase, gitlabBaseUrl } from './gitlab-api.js'; + +interface MergeRequest { + web_url: string; + state: string; + source_branch: string; + target_branch: string; + source_project_id: number; + target_project_id: number; +} + +function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { + return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target + && mr.source_project_id === mr.target_project_id; +} + +async function request(route: string, opts: PrCreateOptions): Promise { + const token = getGitLabToken(); + if (!token) throw new Error('GitLab authentication unavailable. Set GITLAB_TOKEN before resuming.'); + const response = await fetch(`${gitlabApiBase()}/projects/${encodeURIComponent(opts.repo)}/merge_requests${route}`, { + headers: { 'PRIVATE-TOKEN': token, Accept: 'application/json' }, redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitLab MR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('gitlab review lookup returned invalid data.'); }); +} + +export async function findGitLabMergeRequest(opts: PrCreateOptions): Promise { + const query = new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100' }); + const requests = await request(`?${query}`, opts) as MergeRequest[]; + return requests.find((mr) => matches(mr, opts))?.web_url; +} + +export async function validateGitLabMergeRequest(value: string, opts: PrCreateOptions): Promise { + const base = new URL(gitlabBaseUrl()); + const url = new URL(value); + if (url.origin !== base.origin) return undefined; + const prefix = `${base.pathname.replace(/\/$/, '')}/${opts.repo}/-/merge_requests/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as MergeRequest, opts); +} diff --git a/src/providers/tgit/index.ts b/src/providers/tgit/index.ts index da5d724f0..ddc583617 100644 --- a/src/providers/tgit/index.ts +++ b/src/providers/tgit/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import { findTGitMergeRequest, validateTGitMergeRequest } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGfInstalled, @@ -68,6 +69,9 @@ export class TGitProvider implements GitProvider { return 'tencent.com'; } + findOpenPullRequest(opts: PrCreateOptions): Promise { return findTGitMergeRequest(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateTGitMergeRequest(url, opts); } + async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gfListOrgRepos(org, opts); } diff --git a/src/providers/tgit/workflow-pr.ts b/src/providers/tgit/workflow-pr.ts new file mode 100644 index 000000000..96b7c7c36 --- /dev/null +++ b/src/providers/tgit/workflow-pr.ts @@ -0,0 +1,32 @@ +import type { PrCreateOptions } from '../types.js'; +import { tgitFetch } from './rest-auth.js'; + +interface MergeRequest { web_url: string; state: string; source_branch: string; target_branch: string; source_project_id: number; target_project_id: number } +function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { + return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target + && mr.source_project_id === mr.target_project_id; +} +async function list(opts: PrCreateOptions, query: URLSearchParams): Promise { + const response = await tgitFetch(`/projects/${encodeURIComponent(opts.repo)}/merge_requests?${query}`, { + redirect: 'manual', signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`TGit MR lookup failed (${response.status}). Check TGIT_TOKEN or gf authentication.`); + return response.json().catch(() => { throw new Error('TGit MR lookup returned invalid data.'); }) as Promise; +} +export async function findTGitMergeRequest(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const requests = await list(opts, new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100', page: String(page) })); + const found = requests.find((mr) => matches(mr, opts)); + if (found) return found.web_url; + if (requests.length < 100) return undefined; + } + throw new Error('Too many open TGit requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateTGitMergeRequest(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.hostname !== 'git.woa.com') return undefined; + const match = /^\/(.+?)\/(?:-\/)?merge_requests\/(\d+)\/?$/.exec(url.pathname); + if (!match || match[1] !== opts.repo) return false; + const requests = await list(opts, new URLSearchParams({ iid: match[2]! })); + return requests.length === 1 && matches(requests[0]!, opts); +} diff --git a/src/providers/types.ts b/src/providers/types.ts index ccf95ab8b..4b2bae05d 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -120,6 +120,12 @@ export interface GitProvider { */ createPullRequest(opts: PrCreateOptions): Promise; + /** Finds an already open request, so a delivery restarted after a crash does not create another one. */ + findOpenPullRequest?(opts: PrCreateOptions): Promise; + + /** False means known metadata disagrees; undefined means this provider cannot validate the URL. */ + validatePullRequest?(url: string, opts: PrCreateOptions): Promise; + /** * 获取指定 MR/PR 的完整数据（标题、描述、提交列表、diff）。 * diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index 06bf542ff..f8c8f6729 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -28,7 +28,7 @@ export interface ConformanceScenario { * `absent` lists fields the lease must not carry. */ export type ConformanceOperation = - | { kind: 'next'; saveAs: string; running?: string[]; canTakeLease?: boolean; hitl?: boolean; cancelled?: string[]; absent?: string[]; expect: Record } + | { kind: 'next'; saveAs: string; running?: string[]; canTakeLease?: boolean; hitl?: boolean; workspace?: boolean; cancelled?: string[]; absent?: string[]; expect: Record } | { kind: 'respond'; step: string; choice: string; note?: string; pass?: number; json?: unknown; expect: Record } | { kind: 'retry'; step: string; expect: Record } | { kind: 'cancel'; step?: string; expect: Record } @@ -37,6 +37,8 @@ export type ConformanceOperation = lease?: string; attempt?: string; revision?: number; + /** The harness fills the changes branch/base from this operation's saved lease. */ + leasedChanges?: boolean; body: Record; expect: Record; }; @@ -54,6 +56,15 @@ steps: ok: boolean `; +const REVIEW_LINK = `schemaVersion: v1alpha1 +name: review-link +description: A remote driver reports a review link. +steps: + - id: work + workspace: new + command: echo work +`; + const FAILED_DEPENDENCY = `schemaVersion: v1alpha1 name: build description: A step whose dependency failed is skipped. @@ -224,6 +235,23 @@ steps: `; export const workflowConformanceScenarios: ConformanceScenario[] = [ + { + name: 'a driver reports a sanitized review URL', workflowName: 'review-link', yaml: REVIEW_LINK, + operations: [ + { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' } }, expect: { data: { accepted: true, ignored: ['changes.unknown'] } } }, + { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, + ], + expectState: { steps: [{ id: 'work', changes: { pr_url: 'https://review.example/changes/1' } }] }, + }, + { + name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, + operations: [ + { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://user:token@review.example/changes/1' } }, expect: { data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } } }, + { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, + ], + }, { name: 'two-step command success', workflowName: 'build', diff --git a/src/workflow/delivery.ts b/src/workflow/delivery.ts new file mode 100644 index 000000000..2884f7070 --- /dev/null +++ b/src/workflow/delivery.ts @@ -0,0 +1,192 @@ +/** PR delivery is resumable I/O, separate from the agent lease and never under the run lock. */ +import path from 'node:path'; +import type { LocalConfig } from '../types.js'; +import type { GitProvider, PrCreateOptions } from '../providers/types.js'; +import { getProvider, getProviderFromUrl } from '../providers/registry.js'; +import { acquireLock, releaseLock } from '../update.js'; +import { fail, succeed, type StepState } from './step-state.js'; +import { readRun, runLockPath } from './store.js'; +import { withRun, ok } from './engine.js'; +import { sanitizePrUrl } from './pr-url.js'; +import { AGENT_PATHS, git } from './worktree.js'; +import type { StepChanges } from './workspace.js'; +import type { RunState } from './types.js'; +import { publishRunRecords, queueRunRecord } from './records.js'; + +/** Changes in earlier passes remain candidates for a stacked target. */ +function allChanges(run: RunState): StepChanges[] { + return [ + ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), + ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), + ]; +} + +/** Follows the actual copy origin, including after-item precedence, through branches that did not deliver. */ +export function deliveryTarget(run: RunState, branch: string): string | undefined { + const changes = allChanges(run); + const visited = new Set([branch]); + let origin = run.copies?.[branch]?.origin; + while (origin && !visited.has(origin)) { + visited.add(origin); + if (changes.some((item) => item.branch === origin && item.pr_url)) return origin; + origin = run.copies?.[origin]?.origin; + } + return run.base?.ref; +} + +const CI_PATHS = ['.github/workflows', '.github/actions', '.gitlab-ci.yml', '.gitlab', '.coding-ci.yaml', '.cnb', '.circleci', '.buildkite', '.husky', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml']; + +export function protectedDeliveryPaths(files: string[]): string[] { + const roots = [...AGENT_PATHS, ...CI_PATHS].map((root) => root.toLowerCase()); + return files.filter((file) => { + const lower = file.toLowerCase(); + return /(^|\/)(agents|claude)\.md$/.test(lower) + || roots.some((root) => lower === root || lower.startsWith(`${root}/`)); + }); +} + +async function providerFor(run: RunState): Promise<{ provider: GitProvider; opts: PrCreateOptions }> { + if (!run.checkout) throw new Error('The run has no product checkout. Resume from a checkout that can access the delivered branch.'); + // Read the configured URL: `remote get-url` expands insteadOf transport rewrites and can hide the hosting provider. + const remote = await git(run.checkout, ['config', '--get', 'remote.origin.url']); + if (remote.code !== 0) throw new Error('The product checkout has no origin remote. Configure origin before resuming.'); + const url = remote.stdout.trim(); + let hostingUrl = url; + if (/^(https?|ssh):\/\//.test(url)) { + const parsed = new URL(url); + parsed.username = ''; parsed.password = ''; parsed.search = ''; parsed.hash = ''; + hostingUrl = parsed.protocol === 'ssh:' ? `https://${parsed.hostname}${parsed.pathname}` : parsed.toString(); + } else { + const scp = /^[^@\s]+@([^:\s]+):(.+)$/.exec(url); + if (scp) hostingUrl = `https://${scp[1]}/${scp[2]}`; + } + // Filesystem remotes are transport-only too; the registry's bare owner/repo fallback is for user input. + const provider = /^(https?:\/\/|ssh:\/\/|[^@\s]+@[^:\s]+:)/.test(url) ? getProviderFromUrl(hostingUrl) : getProvider('git'); + const repo = provider.name === 'git' ? hostingUrl : (() => { + const parsed = provider.parseRepoInput(hostingUrl); + return `${parsed.owner}/${parsed.repo}`; + })(); + return { provider, opts: { repo, source: '', target: '', title: '', cwd: run.checkout, spawnTimeoutMs: 30_000 } }; +} + +/** Report validation never echoes a rejected value, including a provider's error response. */ +export async function validateReportedPr(run: RunState, changes: StepChanges): Promise<{ url?: string; reason?: string }> { + const clean = sanitizePrUrl(changes.pr_url); + if (!clean.url) return clean; + if (!run.checkout) return clean; + try { + const { provider, opts } = await providerFor(run); + const target = deliveryTarget(run, changes.branch); + if (!target) return { reason: 'run base has no branch' }; + const valid = await provider.validatePullRequest?.(clean.url, { ...opts, source: changes.branch, target }); + return valid === false ? { reason: 'repository or branches do not match the lease' } : clean; + } catch { + return { reason: 'provider metadata could not be validated' }; + } +} + +async function createDelivery(run: RunState, state: Extract): Promise { + const remaining = (): number => { + const ms = state.deliveryDeadline === undefined ? 30_000 : Math.min(30_000, state.deliveryDeadline - Date.now()); + if (ms <= 0) throw new Error('The execution deadline expired before delivery completed.'); + return ms; + }; + remaining(); + const changes = state.changes; + if (!changes || !run.base || !run.checkout) throw new Error('Delivery needs a writable branch, a run base, and its product checkout.'); + const target = deliveryTarget(run, changes.branch); + if (!target) throw new Error('Run base has no branch. Start the run from a named branch (CI checkout ref: main).'); + if (![changes.to, run.base.commit].every((commit) => /^[0-9a-f]{40,64}$/i.test(commit))) throw new Error('Delivery requires full commit object ids.'); + // Do not inspect a later branch head or trust a driver's claimed list of commits. + let head = await git(run.checkout, ['rev-parse', '--verify', `refs/heads/${changes.branch}^{commit}`]); + if (head.code !== 0 || head.stdout.trim() !== changes.to) { + const fetched = await git(run.checkout, ['fetch', '--no-tags', 'origin', `refs/heads/${changes.branch}`]); + head = await git(run.checkout, ['rev-parse', '--verify', 'FETCH_HEAD^{commit}']); + if (fetched.code !== 0 || head.code !== 0 || head.stdout.trim() !== changes.to) throw new Error('The reported branch head is not available. Push the reported commit before resuming.'); + } + const diff = await git(run.checkout, ['diff', '--no-renames', '--name-only', '-z', run.base.commit, changes.to, '--']); + if (diff.code !== 0) throw new Error('Cannot inspect the delivered diff against the run base. Fetch the base commit before resuming.'); + const protectedPaths = protectedDeliveryPaths(diff.stdout.split('\0').filter(Boolean)); + if (protectedPaths.length) throw new Error(`Protected paths changed against the run base: ${protectedPaths.join(', ')}`); + const { provider, opts } = await providerFor(run); + opts.spawnTimeoutMs = remaining(); + if (changes.pr_url) return changes.pr_url; + const request: PrCreateOptions = { ...opts, source: changes.branch, target, + title: `${run.name}: ${state.id}`, description: `Workflow ${run.name}, run ${run.id}, step ${state.id}.\n\nBase: ${run.base.commit}\n\nCreated for human review. TeamAI never merges this request.` }; + const existing = await provider.findOpenPullRequest?.(request); + request.spawnTimeoutMs = remaining(); + let url = existing; + if (!url) { + try { url = await provider.createPullRequest(request); } + catch (error) { + // A provider's response body or CLI output may contain credentials. Keep its established transport-only + // fallback, but never persist arbitrary creation errors from authenticated providers. + if (provider.name === 'git') throw error; + throw new Error(`The ${provider.name} provider could not create the PR. Check its configured credentials, repository/branch access and push permissions, then retry delivery.`); + } + } + const clean = sanitizePrUrl(url); + if (!clean.url) throw new Error('Provider returned an unsafe PR URL. The branch is preserved; inspect the provider before retrying.'); + return clean.url; +} + +/** Drains durable delivery work once. A competing delivery process leaves it for the next poll. */ +export async function deliverPending(config: LocalConfig, runId: string): Promise { + const loaded = await readRun(config, runId); + if (typeof loaded === 'string' || loaded.status !== 'running' || !loaded.steps.some((step) => step.status === 'delivering')) return false; + const lock = path.join(path.dirname(runLockPath(config, runId)), `${runId}.delivery.lock`); + if (!await acquireLock(lock)) return false; + let changed = false; + let queued = false; + try { + for (;;) { + const run = await readRun(config, runId); + if (typeof run === 'string' || run.status !== 'running') break; + const state = run.steps.find((step): step is Extract => step.status === 'delivering'); + if (!state) break; + let url: string | undefined; + let failure: string | undefined; + try { url = await createDelivery(run, state); } + catch (error) { failure = error instanceof Error ? error.message : 'PR delivery failed'; } + const applied = await withRun(config, runId, async (current) => { + const step = current.steps.find((item) => item.id === state.id); + // Cancel/retry may have invalidated this delivery while the provider was running. + if (current.status !== 'running' || step?.status !== 'delivering' || step.pass !== state.pass || step.settledAt !== state.settledAt) { + // An in-flight provider request cannot be revoked. Keep its safe receipt on the matching completed + // attempt/pass, without resurrecting cancelled work, so the end record can still account for the PR. + if (!url || !state.changes) return { save: false, envelope: ok({}) }; + const candidates = [...(step && 'changes' in step ? [step] : []), ...(step?.passes ?? []), + ...(current.archive ?? []).filter((pass) => pass.step === state.id)]; + const receipt = candidates.find((pass) => pass.changes?.branch === state.changes!.branch && pass.changes.to === state.changes!.to + && pass.attempt === state.attempt && (pass.pass ?? 1) === (state.pass ?? 1)); + if (!receipt?.changes) return { save: false, envelope: ok({}) }; + current.revision += 1; + receipt.changes.pr_url = url; + receipt.changes.prRecordedAt ??= current.revision; + if (current.status !== 'running') queued = await queueRunRecord(config, current, current.driver, 'end') || queued; + changed = true; + return { save: true, envelope: ok({}) }; + } + current.revision += 1; + const { status: _status, ...settlement } = step; + const expired = step.deliveryDeadline !== undefined && Date.now() >= step.deliveryDeadline; + if (failure || expired) { + const message = `PR delivery failed for branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved. ${failure}`; + const detail = expired ? `Execution deadline expired while delivering branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved.` : message; + fail(step, { ...settlement, settledAt: current.revision, failure: { code: expired ? 'timeout' : 'delivery_failed', message: detail }, message: detail, + ...(url && step.changes ? { changes: { ...step.changes, pr_url: url, prRecordedAt: step.changes.prRecordedAt ?? current.revision } } : {}) }); + } else { + succeed(step, { ...settlement, settledAt: current.revision, + changes: { ...step.changes!, pr_url: url!, prRecordedAt: step.changes?.prRecordedAt ?? current.revision } }); + } + changed = true; + return { save: true, envelope: ok({}) }; + }); + if ('error' in applied) break; + } + return changed; + } finally { + await releaseLock(lock); + if (queued) await publishRunRecords(config); + } +} diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index bb12cc594..8bffdebfd 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -17,13 +17,14 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - DriverSchema, ReadSchema, answerHitl, cancel, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, + DriverSchema, ReadSchema, answerHitl, cancel, deliver, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { planWorkspace, plannedBranch, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { branchName, planWorkspace, plannedBranch, type StepChanges, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { deliverPending, validateReportedPr } from './delivery.js'; import { generateRunId, isRunId, readRun, runLockPath, writeRun } from './store.js'; import type { CachedWorkflow, @@ -63,7 +64,7 @@ export interface ReportInput { tokens?: TokenUsage; failure?: WorkflowFailure; /** The branch this attempt committed, when it ran in a workspace. */ - changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; + changes?: StepChanges; /** The branch and commit this attempt read, when it ran read-only in a workspace. */ read?: StepRead; agent?: ReportedAgent; @@ -265,8 +266,14 @@ function leaseData( /** Records who opened a workspace the first time this driver takes its branch. */ function rememberCopy(run: RunState, plan: WorkspacePlan, driver: DriverIdentity): void { const branch = plan.contract?.branch; - if (!branch || run.copies?.[branch]) return; - (run.copies ??= {})[branch] = { owner: driver }; + if (!branch) return; + const origin = plan.contract?.start.branch ?? plan.contract?.merge[0]?.branch; + const copy = run.copies?.[branch]; + if (copy) { + if (!copy.origin && !copy.commit && origin && origin !== branch) copy.origin = origin; + return; + } + (run.copies ??= {})[branch] = { owner: driver, ...(origin ? { origin } : {}) }; } /** @@ -510,6 +517,7 @@ export function stopWhere(run: RunState, under: (id: string) => boolean, failure skip(state, settledAt); break; case 'running': + case 'delivering': cancel(state, failure, settledAt); break; case 'leased': @@ -534,14 +542,41 @@ export function stopWhere(run: RunState, under: (id: string) => boolean, failure * A step takes how what it ran settled: a workflow step, the outcome, the end, and the outputs of its workflow; a * `for_each` step, the outcome and the results of its items. */ -function finishStep(state: StepState, result: WorkflowResult): void { +function deliveryDeadline(run: RunState, state: StepState): number | undefined { + const limits: number[] = []; + const root = run.definition as WorkflowDefinition & { timeout?: number }; + if (root.timeout) limits.push(Date.parse(run.createdAt) + root.timeout); + if (state.status === 'leased' && 'deadline' in state.lease && typeof state.lease.deadline === 'number') limits.push(state.lease.deadline); + for (const parent of run.steps) { + if (parent.id !== state.id && !state.id.startsWith(`${parent.id}.`)) continue; + const found = locateStep(run, parent.id); + const scope = found && found.step.workflow !== undefined ? childScope(run, found.scope, statesIn(run, found.scope), parent, found.step) : undefined; + const timeout = (found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout + ?? (scope?.definition as { timeout?: number } | undefined)?.timeout; + if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); + } + return limits.length ? Math.min(...limits) : undefined; +} + +function finishStep(run: RunState, state: StepState, definition: WorkflowStep, result: WorkflowResult): void { const settlement = { settledAt: result.settledAt, ...(result.end !== undefined ? { end: result.end } : {}), ...(result.outputs !== undefined ? { outputs: result.outputs } : {}), ...(result.results ? { results: result.results } : {}), }; - if (result.outcome === 'succeeded') succeed(state, settlement); + if (result.outcome === 'succeeded' && definition.deliver === 'pr' && (!definition.forEach || isItem(state))) { + // A workflow's children may continue one enclosing copy. Deliver that branch once at its workflow boundary. + const own = branchName(run.id, state.id, state.pass ?? 1); + const branches = [...new Set(run.steps.flatMap((child) => child.id.startsWith(`${state.id}.`) && 'changes' in child && child.changes ? [child.changes.branch] : []))]; + const branch = run.copies?.[own] ? own : branches.length === 1 ? branches[0] : undefined; + const copy = branch && run.copies?.[branch]; + if (!branch || !copy || !copy.commit || !run.base) { + fail(state, { ...settlement, failure: { code: 'delivery_failed', message: 'PR delivery needs one writable branch; set workspace: new on the workflow step.' } }); + } else { + deliver(state, { ...settlement, changes: { branch, base: run.base.commit, from: run.base.commit, to: copy.commit, commits: [] } }, deliveryDeadline(run, state)); + } + } else if (result.outcome === 'succeeded') succeed(state, settlement); else fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); } @@ -567,7 +602,7 @@ function advanceInstance( if (state.status === 'running') { const result = settleRunning(run, scope, states, state, definition); if (!('outcome' in result)) return result; - finishStep(state, result); + finishStep(run, state, definition, result); return undefined; } if (definition.forEach && !isItem(state)) { @@ -741,7 +776,7 @@ function settleScope(run: RunState, scope: Scope): Settled { return decided === 'ended' ? result : { ...result, outcome: 'failed', failure: decided }; } if (ready.length > 0) return { ready }; - if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running')) return { inProgress: true }; + if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running' || item.status === 'delivering')) return { inProgress: true }; return { unreadable: `Run ${run.id} has pending steps whose needs cannot be met` }; } } @@ -875,6 +910,12 @@ export interface NextOptions { } export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { + const envelope = await leaseNextCore(config, runId, driver, options); + if ('error' in envelope || !await deliverPending(config, runId)) return envelope; + return leaseNext(config, runId, driver, options); +} + +async function leaseNextCore(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions): Promise { driver = { kind: driver.kind, machine: driver.machine, workspace: driver.workspace }; const running = new Set(options.running); // The end record is queued under the run lock, so the run stores whether it @@ -900,7 +941,9 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv return answer(doneData(run)); }; const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => ( - answer({ wait: { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) + answer({ wait: run.steps.some((step) => step.status === 'delivering') + ? { reason: 'PR delivery is in progress; poll next', delivery: true } + : { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) ); if (run.status !== 'running') { // Settled by an earlier next, which queued the end record unless usage @@ -1097,6 +1140,10 @@ export function parseReport(value: unknown): ParsedReport | string { return 'changes.commits must be an array of strings'; } report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; + if (value.pr_url !== undefined) { + // Invalid types are dropped with a value-free reason by reportAttempt too. + report.changes.pr_url = value.pr_url as string; + } } if (record.read !== undefined) { const read = ReadSchema.safeParse(record.read); @@ -1108,7 +1155,12 @@ export function parseReport(value: unknown): ParsedReport | string { if (typeof agent === 'string') return agent; report.agent = agent; } - return { report, ignored: unreadFields(record, REPORT_FIELDS) }; + const ignored = unreadFields(record, REPORT_FIELDS); + if (isRecord(record.changes)) { + const known = new Set(['branch', 'base', 'from', 'to', 'commits', 'pr_url']); + ignored.push(...Object.keys(record.changes).filter((key) => !known.has(key)).map((key) => `changes.${key}`)); + } + return { report, ignored }; } /** A `next` payload: who the driver is, and what else it tells `next`. */ @@ -1206,7 +1258,20 @@ function branchStillUsed(run: RunState, branch: string): boolean { } export async function reportAttempt(config: LocalConfig, runId: string, report: ReportInput): Promise { - return withRun(config, runId, async (run) => { + const dropped: Array<{ field: string; reason: string }> = []; + if (report.changes && report.changes.pr_url !== undefined) { + report = structuredClone(report); + const run = await readRun(config, runId); + if (typeof run !== 'string') { + const result = await validateReportedPr(run, report.changes!); + if (result.url) report.changes!.pr_url = result.url; + else { + delete report.changes!.pr_url; + dropped.push({ field: 'changes.pr_url', reason: result.reason! }); + } + } + } + const envelope = await withRun(config, runId, async (run) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { return { save: false, envelope: error('repeated_report', `Attempt ${report.attempt} was already reported`) }; @@ -1266,7 +1331,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: model: report.model, session: report.session, tokens: report.tokens, - ...(report.changes ? { changes: report.changes } : {}), + ...(report.changes ? { changes: { ...report.changes, prRecordedAt: report.changes.pr_url ? run.revision : undefined } } : {}), ...(report.read ? { read: report.read } : {}), agent: settleObservation(report, defined?.agent), }; @@ -1275,11 +1340,12 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: copy.commit = report.changes.to; run.copies[report.changes.branch] = copy; } + const successful = { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }; const settledStep = outcome === 'released' ? release(step) - : outcome === 'success' ? succeed(step, { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }) + : outcome === 'success' ? defined?.deliver === 'pr' ? deliver(step, successful, deliveryDeadline(run, step)) : succeed(step, successful) : fail(step, { ...settlement, failure }); // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. - const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); + const cleanup = outcome === 'success' && settledStep.status !== 'delivering' && contract !== undefined && !branchStillUsed(run, contract.branch); return { save: true, envelope: ok({ @@ -1290,4 +1356,22 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: }), }; }); + if ('error' in envelope) return envelope; + const pending = await readRun(config, runId); + if (typeof pending !== 'string' && pending.status === 'running' && pending.steps.some((step) => step.status === 'delivering')) { + // Reconcile before provider I/O too: limits reached by this report must stop delivery before it opens a PR. + const checked = await leaseNextCore(config, runId, report.driver, { canTakeLease: false, + running: pending.steps.flatMap((step) => step.status === 'leased' ? [step.attempt] : []), + }); + if (!('error' in checked)) await deliverPending(config, runId); + const run = await readRun(config, runId); + const step = typeof run !== 'string' && run.steps.find((state) => 'attempt' in state && state.attempt === report.attempt); + if (step) { + envelope.data.revision = run.revision; + envelope.data.step = { id: step.id, status: step.status, outcome: step.status === 'failed' || step.status === 'cancelled' ? 'failed' : 'success' }; + if (step.status === 'succeeded' && report.changes && !branchStillUsed(run, report.changes.branch)) envelope.data.cleanup = true; + } + } + if (dropped.length) envelope.data.dropped = dropped; + return envelope; } diff --git a/src/workflow/fan-out.ts b/src/workflow/fan-out.ts index 459339c29..cd1662fcb 100644 --- a/src/workflow/fan-out.ts +++ b/src/workflow/fan-out.ts @@ -95,7 +95,7 @@ export function afterStanding(after: StepState[]): 'ready' | 'waiting' | 'skip' /** How many items are started and not yet settled, which `max_parallel` bounds. */ export function startedItems(items: StepState[]): number { - return items.filter((item) => item.status === 'leased' || item.status === 'running').length; + return items.filter((item) => item.status === 'leased' || item.status === 'running' || item.status === 'delivering').length; } /** diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 421be7d6f..38905ff3b 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -946,6 +946,10 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options if ('lease' in next.data && room()) continue; if (inFlight.size === 0 && dying.size === 0 && !prompt && !question) { // A wait for a call that could not take a lease says nothing about what is left; ask again with room. + if ((next.data.wait as { delivery?: boolean } | undefined)?.delivery) { + await new Promise((resolve) => setTimeout(resolve, 100)); + continue; + } if (!canTakeLease) continue; const reason = typeof (next.data.wait as { reason?: unknown }).reason === 'string' ? (next.data.wait as { reason: string }).reason diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 244467c43..998b7c79d 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -19,7 +19,7 @@ import { loopProblems } from './loop.js'; const HEADER_KEYS = new Set(['schemaVersion', 'name', 'description', 'inputs', 'defaults', 'ends', 'outputs', 'loop', 'steps']); const ACTION_KEYS = Object.keys(STEP_ACTIONS); -const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session']); +const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session', 'deliver']); const STEP_LOOP_KEYS = new Set(['back_to', 'while', 'max_iterations']); const FOR_EACH_KEYS = new Set(['items', 'max_parallel']); const WORKFLOW_LOOP_KEYS = new Set(['while', 'max_iterations']); @@ -491,6 +491,14 @@ function parseBody( }); } const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); + const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); + const deliver = scalarString(deliverPair?.value)?.trim(); + if (deliverPair && (deliver !== 'pr' || workspace === 'read-only')) { + diagnostics.push({ + ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, 'deliver must be pr on a writable branch'), + code: 'bad_delivery', + }); + } let session: WorkflowStep['session']; if (sessionPair) { const sessionText = scalarString(sessionPair.value)?.trim(); @@ -537,6 +545,7 @@ function parseBody( ...(breakSource !== undefined ? { break: breakSource } : {}), ...(forEach ? { forEach } : {}), ...(workspace ? { workspace } : {}), + ...(deliver === 'pr' ? { deliver } : {}), ...(session ? { session } : {}), }); } diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts new file mode 100644 index 000000000..02ee366ae --- /dev/null +++ b/src/workflow/pr-url.ts @@ -0,0 +1,16 @@ +/** Public run records accept HTTPS review links on any host, but never URL credentials or session parameters. */ +export function sanitizePrUrl(value: unknown): { url: string; reason?: never } | { url?: never; reason: string } { + if (typeof value !== 'string') return { reason: 'must be a URL string' }; + if (value.length > 512) return { reason: 'exceeds 512 characters' }; + try { + const url = new URL(value); + if (url.protocol !== 'https:') return { reason: 'must use HTTPS' }; + if (url.username || url.password) return { reason: 'contains credentials' }; + url.search = ''; + url.hash = ''; + const sanitized = url.toString(); + return sanitized.length <= 512 ? { url: sanitized } : { reason: 'exceeds 512 characters' }; + } catch { + return { reason: 'is not a valid URL' }; + } +} diff --git a/src/workflow/records.ts b/src/workflow/records.ts index 85124560d..06166a6ec 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -25,6 +25,7 @@ import { stepDefinitions } from './scope.js'; import { runLockPath } from './store.js'; import { settled, VISIBLE_OUTCOMES, type StepPass, type StepState, type VisibleOutcome } from './step-state.js'; import type { DriverIdentity, RunState, RunStatus, TokenUsage, WorkflowFailure } from './types.js'; +import { sanitizePrUrl } from './pr-url.js'; /** * The failure codes the engine writes, then those the local driver sets. A code a model writes into its result, or @@ -35,6 +36,7 @@ const RECORDED_FAILURES = [ 'loop_exhausted', 'invalid_output', 'invalid_template', 'invalid_inputs', 'invalid_items', 'item_cycle', 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', + 'delivery_failed', 'timeout', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; @@ -94,6 +96,8 @@ export type WorkflowRunRecord = { stepCounts?: { total: number; succeeded: number; failed: number; skipped: number; cancelled: number }; /** Settled attempts across the run — the cumulative iteration count. */ iterations?: number; + /** Validated review links, first appearance order. Public on a public team repo; omitted before this version. */ + prs?: string[]; username: string; publishedAt: string; }; @@ -207,6 +211,13 @@ export function buildEndRecord( ]), ...(run.archive ?? []).map((archived) => entry(archived.step, archived.pass, archived)), ]; + const changes = [ + ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), + ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), + ]; + const prs = [...new Set(changes.filter((item) => item.prRecordedAt !== undefined) + .sort((a, b) => a.prRecordedAt! - b.prRecordedAt!) + .flatMap((item) => sanitizePrUrl(item.pr_url).url ?? []))]; return { ...buildStartRecord(run, driver, username, now), phase: 'end', @@ -228,6 +239,7 @@ export function buildEndRecord( // Cumulative when present; runs from before the counter fall back to the // bounded history length (their true count is unknowable now). iterations: run.settledCount ?? run.settledAttempts.length, + ...(prs.length ? { prs } : {}), }; } diff --git a/src/workflow/run-controls.ts b/src/workflow/run-controls.ts index 67a53de41..a01487de3 100644 --- a/src/workflow/run-controls.ts +++ b/src/workflow/run-controls.ts @@ -8,7 +8,7 @@ import { isItem, itemsAfter, itemsOf } from './fan-out.js'; import { archiveUnder, isUnder } from './loop.js'; import { publishRunRecords, removeQueuedEndRecord } from './records.js'; import { contextOf, localId, statesIn } from './scope.js'; -import { answerHitl, cancel, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; +import { answerHitl, cancel, deliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; import type { RunState, WorkflowEnvelope } from './types.js'; async function controlRun( @@ -110,7 +110,7 @@ export async function cancelRun(config: LocalConfig, runId: string, step?: strin } else { const state = run.steps.find((state) => state.id === step); if (!state) return missingStepError(run, step); - if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased or running steps can be cancelled`); + if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running' && state.status !== 'delivering') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased, running or delivering steps can be cancelled`); const located = locateStep(run, step); if (!located) return missingStepError(run, step); changed = state.status === 'running' ? stopWhere(run, isUnder(state.id), failure, run.revision) : []; @@ -219,6 +219,7 @@ function retryInstance(run: RunState, state: StepState): void { cancel(state, failure, run.revision); break; case 'running': + case 'delivering': stopWhere(run, isUnder(state.id), failure, run.revision); break; case 'succeeded': @@ -259,7 +260,10 @@ export async function retryStep(config: LocalConfig, runId: string, step: string const reopened = run.status !== 'running'; for (const container of containers) { if (container.child === target) { - retryInstance(run, target); + if (target.status === 'failed' && target.failure?.code === 'delivery_failed' && target.changes) { + const { status: _status, outcome: _outcome, failure: _failure, message: _message, ...settlement } = target; + deliver(target, { ...settlement, settledAt: run.revision + 1 }); + } else retryInstance(run, target); resetIteration(target); } for (const state of container.states) { diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index f249d9864..1ec6a6f32 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -21,6 +21,8 @@ const ChangesSchema = z.object({ from: z.string(), to: z.string(), commits: z.array(z.string()), + pr_url: z.string().optional(), + prRecordedAt: z.number().int().optional(), }); /** What a read-only step inside a copy reports: the branch it ran on and the commit it read. It wrote nothing. */ @@ -119,6 +121,8 @@ const settledFields = { results: ResultsSchema.optional(), settledAt: z.number().int().default(0), attempt: z.string().optional(), + /** The original execution bound continues while its completed work is delivered. */ + deliveryDeadline: z.number().optional(), ...report, }; @@ -133,12 +137,13 @@ export const StepStateSchema = z.discriminatedUnion('status', [ StepBaseSchema.extend({ status: z.literal('waiting') }), StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }), StepBaseSchema.extend({ status: z.literal('running') }), + StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields }), StepBaseSchema.extend({ status: z.literal('succeeded'), outcome: z.literal('success').default('success'), ...settledFields, loopFinal: z.boolean().optional(), }), StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional() }), StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }), - StepBaseSchema.extend({ status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema }), + StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }), ]); type ReadonlyId = T extends unknown ? Omit & { readonly id: string } : never; /** A step instance's `id` is read-only: the engine indexes instances by it, and every transition keeps it. */ @@ -215,13 +220,22 @@ export function succeed(state: StepState, settlement: Settlement) return become(state, { status: 'succeeded', outcome: 'success', ...settlement }); } +/** Agent work is durable, but dependents wait for delivery to finish. */ +export function deliver(state: StepState, settlement: Settlement, deadline?: number): StepIn<'delivering'> { + return become>(state, { status: 'delivering', ...settlement, ...(deadline !== undefined ? { deliveryDeadline: deadline } : {}) }); +} + export function fail(state: StepState, settlement: Settlement): FailedStep { return become(state, { status: 'failed', outcome: 'failed', ...settlement }); } /** An open step is cancelled before it settles. A leased one keeps its invalidated attempt. */ -export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running'>, failure: WorkflowFailure, settledAt: number): CancelledStep { - return become(state, { status: 'cancelled', settledAt, ...(state.status === 'leased' ? { attempt: state.attempt } : {}), failure }); +export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { + // Delivery follows completed agent work. Cancellation must retain that work's usage and any review link. + const reported = state.status === 'delivering' + ? Object.fromEntries(SETTLED_KEYS.filter((key) => state[key] !== undefined).map((key) => [key, state[key]])) + : state.status === 'leased' ? { attempt: state.attempt } : {}; + return become(state, { ...reported, status: 'cancelled', settledAt, failure }); } /** The current pass of a settled step, as `passes` keeps it once a loop runs the step again; undefined while it has not settled. */ @@ -279,6 +293,7 @@ export function stopped(step: StepState): boolean { case 'waiting': case 'leased': case 'running': + case 'delivering': case 'succeeded': case 'skipped': return false; @@ -302,6 +317,7 @@ export function settled(step: StepState | undefined): step is Extract; + copies?: Record; createdAt: string; updatedAt: string; } diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index eb691f8ef..d704e680f 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -17,6 +17,9 @@ export interface StepChanges { from: string; to: string; commits: string[]; + pr_url?: string; + /** Engine revision at which the validated URL first appeared. Never accepted from a driver. */ + prRecordedAt?: number; } /** @@ -114,13 +117,26 @@ function workflowChildren(run: RunState, step: StepState): StepState[] { /** Branches a step merges, `after` items first, then its other needs. An omitted `needs` uses the parent step's needs. */ function mergeChanges(run: RunState, scope: Scope, state: StepState, definition: WorkflowStep): StepChanges[] { const ordered: Array<{ changes: StepChanges; settledAt: number }> = []; + const visited = new Set(); const push = (step: StepState | undefined): void => { - if (!step) return; + if (!step || visited.has(step.id)) return; + visited.add(step.id); const changes = changesOf(step); // A for_each step keeps each item's branch on that item, and a workflow step keeps each child's on that child. if (!changes) { for (const item of itemsOf(run, step)) push(item); for (const child of workflowChildren(run, step)) push(child); + // A skipped or read-only need wrote no branch, but its earlier dependencies still supply the copy's origin. + let id: string | undefined = step.id; + while (id) { + const defined = stepDefinitions(run)(id); + const prefix: string = id.includes('.') ? id.slice(0, id.lastIndexOf('.') + 1) : ''; + if (defined?.needs !== undefined) { + for (const need of defined.needs) push(run.steps.find((item) => item.id === `${prefix}${need}`)); + break; + } + id = prefix ? prefix.slice(0, -1) : undefined; + } return; } const settledAt = 'settledAt' in step ? step.settledAt : 0; From 6056c1563a5c404980b3ea9b3fa5e46a10a60852 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:25:50 +0800 Subject: [PATCH 02/77] fix(workflow): retain every enclosing delivery timeout - Carry both a workflow step timeout and its nested workflow timeout into the delivery deadline, keeping the earliest bound rather than preferring one. - Keep the root and lease deadlines in the same minimum so delivery cannot relax the constraints established by the requirements implementation. - Verify all 29 delivery regressions, typecheck, lint and the build-first real CLI PR creation/reuse, generic-host recovery and required-approval cases. --- src/workflow/engine.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 8bffdebfd..3e96f3c1c 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -551,9 +551,9 @@ function deliveryDeadline(run: RunState, state: StepState): number | undefined { if (parent.id !== state.id && !state.id.startsWith(`${parent.id}.`)) continue; const found = locateStep(run, parent.id); const scope = found && found.step.workflow !== undefined ? childScope(run, found.scope, statesIn(run, found.scope), parent, found.step) : undefined; - const timeout = (found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout - ?? (scope?.definition as { timeout?: number } | undefined)?.timeout; - if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); + const bounds = [(found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout, + (scope?.definition as { timeout?: number } | undefined)?.timeout]; + for (const timeout of bounds) if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); } return limits.length ? Math.min(...limits) : undefined; } From f6490bcdfb919fb84e9108ee0aa00b3ccad1b794 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:31:56 +0800 Subject: [PATCH 03/77] fix(workflow): recheck cancellation before opening review requests - Reload the durable run and verify the same delivering pass/attempt after an open-request lookup returns no match, before starting any PR creation request. Cancellation during lookup now prevents creation; already in-flight creation still retains its safe receipt without reviving cancelled work. - Reject deliver: pr on commands with an implicitly read-only workspace, while retaining prompts that inherit a writable enclosing workflow. Clarify the command requirement in both language guides. - Add a regression that cancels during lookup and previously observed an unwanted create call, plus parser coverage for implicit read-only commands and valid inherited prompt workspaces. - Verify 118 delivery/parser/provider/guide tests, typecheck, lint and a build-first real CLI delivery, reuse, recovery and approval round trip. --- docs/guide/workflow.md | 2 ++ docs/guide/zh-CN/workflow.md | 2 ++ src/__tests__/workflow-delivery.test.ts | 15 +++++++++++++++ src/workflow/delivery.ts | 12 +++++++++--- src/workflow/parse.ts | 3 ++- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 11b884ffc..5eb98a688 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -216,6 +216,8 @@ steps: The PR targets the nearest delivered branch along the branch's recorded origin; `for_each.after` comes before other `needs`, and the first child inherits the enclosing workflow's dependencies. Skipped or read-only dependencies are passed over. If no earlier branch delivered, the target is the run's original branch. Start from a named branch: a detached checkout has no PR target. An open PR for the same head and target is reused. +Commands that deliver must explicitly choose `workspace: new` or `continue`: omitting workspace makes a command read-only even inside a writable workflow. Prompts can inherit an enclosing writable workspace. + Agent completion is followed by a durable `delivering` state. Dependents wait for it. Git and provider operations happen outside the run lock. A PR creation failure reports `delivery_failed` and names the preserved branch. Fix provider authentication, run `teamai workflow retry `, then `teamai workflow resume `: this retries delivery on that branch without rerunning the agent. Push failures remain `push_failed` and retain the worktree. A completed composite may already have removed its worktree; its committed, pushed branch remains. Generic Git hosts support transport only: create a PR yourself and report its URL, or use a provider that supports PR creation. No interactive login starts during delivery. Before opening a PR, TeamAI checks the delivered commit against the run's base commit. Changes to CI, hooks, agent instructions, rules, MCP and equivalent tool configuration are rejected, with the paths named. The shared agent configuration paths are the same ones restored before workspace execution. This check includes changes inherited from earlier branches and deletions/renames. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index d059afcab..4106b1fba 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -216,6 +216,8 @@ steps: PR 的目标沿分支记录的来源，追溯到最近已经交付的分支。`for_each.after` 优先于其他 `needs`，第一个子步骤继承外层工作流的依赖；跳过或只读的依赖会继续向前追溯。之前没有交付分支时，目标为运行开始时的分支。因此请从有名称的分支开始；detached checkout 没有 PR 目标。同一源分支和目标分支已有打开的 PR 时会复用。 +交付 PR 的命令必须显式选择 `workspace: new` 或 `continue`；即使位于可写工作流中，省略 workspace 的命令仍为只读。提示词可以继承外层的可写工作区。 + agent 完成后，步骤进入持久的 `delivering` 状态，后续依赖等待交付结束。Git 和 provider 操作不占用运行锁。创建 PR 失败时以 `delivery_failed` 结束，消息写明保留的分支。修好认证后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，只重试这条分支的交付，不重跑 agent。推送失败仍为 `push_failed`，并保留 worktree。组合工作流结束时可能已移除 worktree，但提交和远程分支仍在。通用 Git 主机只支持传输：请自行创建 PR 并上报 URL，或使用支持创建 PR 的 provider。交付不会启动交互登录。 创建 PR 前，TeamAI 将交付提交与运行的基准提交比较。CI、hooks、agent 指令、rules、MCP 和同类工具配置的修改会被拒绝，并列出路径。共享 agent 配置路径与工作区执行前恢复的路径相同。检查也覆盖从前序分支继承的修改，以及删除和重命名。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 7ab3666af..c1167c8d1 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -77,6 +77,10 @@ describe('workflow PR delivery contract', () => { const parsed = parseWorkflow('ship.yaml', yaml); expect(parsed.ok && parsed.definition.steps[0]).toMatchObject({ deliver: 'pr' }); expect(parseWorkflow('ship.yaml', yaml.replace('deliver: pr', 'deliver: merge')).ok).toBe(false); + expect(parseWorkflow('ship.yaml', yaml.replace(' workspace: new\n', '')).ok).toBe(false); + const inherited = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n prompt: Edit the code.\n deliver: pr\n'; + expect(parseWorkflow('ship.yaml', inherited).ok).toBe(true); + expect(parseWorkflow('ship.yaml', inherited.replace('prompt: Edit the code.', 'command: echo edit')).ok).toBe(false); }); it('keeps the URL for validation and names unread nested fields without values', () => { @@ -269,6 +273,17 @@ describe('durable PR delivery', () => { expect(run.steps[0]).toMatchObject({ status: 'cancelled', changes: { pr_url: 'https://github.com/fixture/app/pull/99' } }); expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/99']); }); + it('does not create a PR when cancellation completed during the open-request lookup', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + api.lookup.mockImplementation(async () => { + await cancelRun(config, runId, 'write'); + return undefined; + }); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect(api.create).not.toHaveBeenCalled(); + expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + }); it('keeps an execution deadline across delivery and makes no provider call once it expired', async () => { const api = provider(); diff --git a/src/workflow/delivery.ts b/src/workflow/delivery.ts index 2884f7070..b24d9ecbc 100644 --- a/src/workflow/delivery.ts +++ b/src/workflow/delivery.ts @@ -85,7 +85,7 @@ export async function validateReportedPr(run: RunState, changes: StepChanges): P } } -async function createDelivery(run: RunState, state: Extract): Promise { +async function createDelivery(config: LocalConfig, run: RunState, state: Extract): Promise { const remaining = (): number => { const ms = state.deliveryDeadline === undefined ? 30_000 : Math.min(30_000, state.deliveryDeadline - Date.now()); if (ms <= 0) throw new Error('The execution deadline expired before delivery completed.'); @@ -114,9 +114,15 @@ async function createDelivery(run: RunState, state: Extract step.id === state.id); + if (typeof latest === 'string' || latest.status !== 'running' || !active || active.status !== 'delivering' + || active.pass !== state.pass || active.attempt !== state.attempt || active.settledAt !== state.settledAt) { + throw new Error('Delivery was cancelled or replaced during PR lookup; no PR was created.'); + } + request.spawnTimeoutMs = remaining(); try { url = await provider.createPullRequest(request); } catch (error) { // A provider's response body or CLI output may contain credentials. Keep its established transport-only @@ -146,7 +152,7 @@ export async function deliverPending(config: LocalConfig, runId: string): Promis if (!state) break; let url: string | undefined; let failure: string | undefined; - try { url = await createDelivery(run, state); } + try { url = await createDelivery(config, run, state); } catch (error) { failure = error instanceof Error ? error.message : 'PR delivery failed'; } const applied = await withRun(config, runId, async (current) => { const step = current.steps.find((item) => item.id === state.id); diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 998b7c79d..a7ba72b1d 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -493,7 +493,8 @@ function parseBody( const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); const deliver = scalarString(deliverPair?.value)?.trim(); - if (deliverPair && (deliver !== 'pr' || workspace === 'read-only')) { + // A command without workspace is always read-only, including inside a writable workflow. + if (deliverPair && (deliver !== 'pr' || workspace === 'read-only' || (commandPair && workspace === undefined))) { diagnostics.push({ ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, 'deliver must be pr on a writable branch'), code: 'bad_delivery', From 7e6bef55b33b7d9c667d93efcc88edc77c0b31b9 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 10:56:01 +0200 Subject: [PATCH 04/77] refactor(workflow): derive step status lists from one source The snapshot union was a hand-kept copy of the run union, so a status added to one and missed in the other compiled and then failed every publish with snapshot_unsupported. Both unions now come from one variant list; only the leased variant differs. The in-progress check in settleScope and the cancellable check in cancel were hand-written status chains whose miss fails at run time (run_unreadable, not_cancellable). They are now exhaustive predicates, so a new status does not compile until each one places it. Red: "a pending step is in progress: false, and can be cancelled: true" failed with "(0 , inProgress) is not a function". Dropping a status from the snapshot union makes the parity test fail in tsc (TS2344) and at run time ("expected [ 'cancelled', 'failed', ... ] to deeply equal [ ..., 'delivering', ... ]"). --- src/__tests__/workflow-step-state.test.ts | 30 ++++++++ src/workflow/engine.ts | 4 +- src/workflow/run-controls.ts | 4 +- src/workflow/step-state.ts | 83 +++++++++++++++-------- 4 files changed, 89 insertions(+), 32 deletions(-) create mode 100644 src/__tests__/workflow-step-state.test.ts diff --git a/src/__tests__/workflow-step-state.test.ts b/src/__tests__/workflow-step-state.test.ts new file mode 100644 index 000000000..be267d918 --- /dev/null +++ b/src/__tests__/workflow-step-state.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, expectTypeOf, it } from 'vitest'; +import { + StepCheckpointSchema, StepStateSchema, cancellable, inProgress, type StepCheckpoint, type StepState, +} from '../workflow/step-state.js'; + +function statusesOf(schema: typeof StepStateSchema | typeof StepCheckpointSchema): string[] { + return schema.options.map((option) => option.shape.status.value).sort(); +} + +const step = (status: StepState['status']): StepState => StepStateSchema.parse({ + id: 'work', status, + ...(status === 'leased' ? { attempt: 'a1', lease: { attempt: 'a1', revision: 1, driver: { kind: 'local', machine: 'm', workspace: 'w' } } } : {}), + ...(status === 'cancelled' ? { failure: { code: 'cancelled', message: 'Cancelled' } } : {}), +}); + +describe('step statuses', () => { + it('a handed-over run carries every status a local run has', () => { + expectTypeOf().toEqualTypeOf(); + expect(statusesOf(StepCheckpointSchema)).toEqual(statusesOf(StepStateSchema)); + }); + + it.each([ + ['pending', false, true], ['waiting', true, true], ['leased', true, true], ['running', true, true], + ['delivering', true, true], ['succeeded', false, false], ['failed', false, false], ['skipped', false, false], + ['cancelled', false, false], + ] as const)('a %s step is in progress: %s, and can be cancelled: %s', (status, open, stoppable) => { + expect(inProgress(step(status))).toBe(open); + expect(cancellable(step(status))).toBe(stoppable); + }); +}); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 9fef9783d..2d5cbd054 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -18,7 +18,7 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - ReadSchema, answerHitl, cancel, deliver, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, + ReadSchema, answerHitl, cancel, deliver, fail, inProgress, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; @@ -775,7 +775,7 @@ function settleScope(run: RunState, scope: Scope): Settled { return decided === 'ended' ? result : { ...result, outcome: 'failed', failure: decided }; } if (ready.length > 0) return { ready }; - if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running' || item.status === 'delivering')) return { inProgress: true }; + if (states.some(inProgress)) return { inProgress: true }; return { unreadable: `Run ${run.id} has pending steps whose needs cannot be met` }; } } diff --git a/src/workflow/run-controls.ts b/src/workflow/run-controls.ts index b1bcb84f1..ac68f95db 100644 --- a/src/workflow/run-controls.ts +++ b/src/workflow/run-controls.ts @@ -9,7 +9,7 @@ import { publishRunEnd } from './handoff.js'; import { archiveUnder, isUnder } from './loop.js'; import { publishRunRecords, removeQueuedEndRecord } from './records.js'; import { contextOf, localId, statesIn } from './scope.js'; -import { answerHitl, cancel, deliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; +import { answerHitl, cancel, cancellable, deliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; import type { RunState, WorkflowEnvelope } from './types.js'; async function controlRun( @@ -119,7 +119,7 @@ export async function cancelRun(config: LocalConfig, runId: string, step?: strin } else { const state = run.steps.find((state) => state.id === step); if (!state) return missingStepError(run, step); - if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running' && state.status !== 'delivering') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased, running or delivering steps can be cancelled`); + if (!cancellable(state)) return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased, running or delivering steps can be cancelled`); const located = locateStep(run, step); if (!located) return missingStepError(run, step); changed = state.status === 'running' ? stopWhere(run, isUnder(state.id), failure, run.revision) : []; diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 92a7b09d1..89ace6749 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -161,38 +161,23 @@ const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcom const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }); -export const StepStateSchema = z.discriminatedUnion('status', [ - PendingSchema, - WaitingSchema, - StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }), - RunningSchema, - DeliveringSchema, - SucceededSchema, - FailedSchema, - SkippedSchema, - CancelledSchema, -]); -type ReadonlyId = T extends unknown ? Omit & { readonly id: string } : never; -/** A step instance's `id` is read-only: the engine indexes instances by it, and every transition keeps it. */ -export type StepState = ReadonlyId>; - +const LeasedSchema = StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }); /** * A step as a run handed to another machine carries it. A leased step keeps its attempt and the kind of driver that * holds it, not that driver's machine or workspace, which mean nothing on the machine that takes the run. */ -export const StepCheckpointSchema = z.discriminatedUnion('status', [ - PendingSchema, - WaitingSchema, - StepBaseSchema.extend({ - status: z.literal('leased'), attempt: z.string(), lease: z.object({ attempt: z.string(), revision: z.number(), driverKind: z.string() }), - }), - RunningSchema, - DeliveringSchema, - SucceededSchema, - FailedSchema, - SkippedSchema, - CancelledSchema, -]); +const HandedLeaseSchema = StepBaseSchema.extend({ + status: z.literal('leased'), attempt: z.string(), lease: z.object({ attempt: z.string(), revision: z.number(), driverKind: z.string() }), +}); +/** Every status but `leased` is the same locally and in a snapshot, so a status added here reaches both unions. */ +const SHARED_VARIANTS = [PendingSchema, WaitingSchema, RunningSchema, DeliveringSchema, SucceededSchema, FailedSchema, SkippedSchema, CancelledSchema] as const; + +export const StepStateSchema = z.discriminatedUnion('status', [LeasedSchema, ...SHARED_VARIANTS]); +type ReadonlyId = T extends unknown ? Omit & { readonly id: string } : never; +/** A step instance's `id` is read-only: the engine indexes instances by it, and every transition keeps it. */ +export type StepState = ReadonlyId>; + +export const StepCheckpointSchema = z.discriminatedUnion('status', [HandedLeaseSchema, ...SHARED_VARIANTS]); export type StepCheckpoint = ReadonlyId>; /** A lease another machine held when the run was handed over. */ export type HandedLease = Extract; @@ -379,6 +364,48 @@ export function stopped(step: StepState): boolean { } } +/** A step a driver or the engine is still working on, so its workflow waits rather than being stuck. */ +export function inProgress(step: StepState): boolean { + switch (step.status) { + case 'waiting': + case 'leased': + case 'running': + case 'delivering': + return true; + case 'pending': + case 'succeeded': + case 'failed': + case 'skipped': + case 'cancelled': + return false; + default: { + const unknown: never = step; + return unknown; + } + } +} + +/** A step `teamai workflow cancel` may stop: one that has not settled. */ +export function cancellable(step: StepState): step is StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'> { + switch (step.status) { + case 'pending': + case 'waiting': + case 'leased': + case 'running': + case 'delivering': + return true; + case 'succeeded': + case 'failed': + case 'skipped': + case 'cancelled': + return false; + default: { + const unknown: never = step; + return unknown; + } + } +} + /** A step that will not run again in this pass, so steps that need or read it can go on. */ export function settled(step: StepState | undefined): step is Extract { if (!step) return false; From cd1d638eed7e578c3fdfd82975e8e4af1d2ce6c8 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 11:01:00 +0200 Subject: [PATCH 05/77] feat(workflow): refuse deliver: pr where nothing writes a branch A top-level prompt with deliver: pr and no workspace runs read-only, so it passed validation and then failed at delivery after the agent had spent its tokens. One file cannot decide this: the same prompt inside a sub-workflow continues the copy of the workspace: new step that runs it. workflow start now resolves each delivering step's effective workspace on the whole tree and refuses with bad_delivery, naming the step. A workflow step that delivers must own its branch with workspace: new. A pull request targets a branch, and a run started from a detached HEAD has none, so a tree that delivers is refused at start with no_base_branch, telling to check out a named branch. Per-file diagnostics keep what no enclosing step can change (a value other than pr, explicit read-only, a bare command) and now name the step and the fix. Red: "refuses a top-level prompt that delivers but runs read-only" and "refuses a run that delivers from a detached HEAD" failed with "expected { data: { ... } } to deeply equal { error: { ... } }": the run started. The parse case failed on the old message "deliver must be pr on a writable branch". --- src/__tests__/workflow-engine.test.ts | 76 ++++++++++++++++++++++++++ src/__tests__/workflow-parse.test.ts | 30 ++++++++++ src/__tests__/workflow-records.test.ts | 4 +- src/workflow/engine.ts | 4 +- src/workflow/parse.ts | 11 +++- src/workflow/workspace.ts | 37 ++++++++++++- 6 files changed, 156 insertions(+), 6 deletions(-) diff --git a/src/__tests__/workflow-engine.test.ts b/src/__tests__/workflow-engine.test.ts index d678c3f75..a85aa4ff0 100644 --- a/src/__tests__/workflow-engine.test.ts +++ b/src/__tests__/workflow-engine.test.ts @@ -4413,3 +4413,79 @@ describe('the session a step resumes after other steps continued it', () => { expect(parseReport({ ...report, resumed: { tool: 'claude', id: '' } })).toBe('resumed must be a session: a tool and an id'); }); }); + +describe('deliver: pr at workflow start', () => { + async function start(files: Record, name: string, base?: RunState['base']): Promise<{ config: LocalConfig; started: Awaited> }> { + const { config } = testConfig(); + for (const [file, body] of Object.entries(files)) fs.writeFileSync(path.join(config.repo.localPath, 'workflows', `${file}.yaml`), body); + await syncWorkflowDefinitions(config); + return { config, started: await startRun(config, name, {}, base ? { base } : {}) }; + } + const commit = 'a'.repeat(40); + + it('refuses a top-level prompt that delivers but runs read-only, naming the step, before the run exists', async () => { + const { config, started } = await start({ ship: workflow('ship', ` - id: fix + prompt: Fix the bug. + deliver: pr +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toEqual({ error: { + code: 'bad_delivery', + message: 'Step "fix" has deliver: pr but runs read-only. Set workspace: new on it, or on the workflow step that runs it.', + } }); + expect(fs.existsSync(path.join(config.dataHome ?? '', 'workflows', 'runs'))).toBe(false); + }); + + it('starts the same prompt inside a sub-workflow that a workspace: new step runs', async () => { + const { started } = await start({ + fix: workflow('fix', ` - id: write + prompt: Fix the bug. + deliver: pr +`), + ship: workflow('ship', ` - id: change + workspace: new + workflow: fix +`), + }, 'ship', { ref: 'main', commit }); + + expect(started).toMatchObject({ data: { status: 'running' } }); + }); + + it('refuses a workflow step that delivers without a copy of its own', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: change + deliver: pr + workflow: + steps: + - id: write + workspace: new + prompt: Fix the bug. +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toEqual({ error: { + code: 'bad_delivery', + message: 'Step "change" delivers its workflow, so it needs workspace: new to own the branch it delivers.', + } }); + }); + + it('refuses a run that delivers from a detached HEAD, telling to start from a named branch', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: fix + workspace: new + prompt: Fix the bug. + deliver: pr +`) }, 'ship', { commit }); + + expect(started).toEqual({ error: { + code: 'no_base_branch', + message: `Step "fix" delivers a pull request, but the run would start from a detached HEAD at ${commit}, so no branch is there for the pull request to target. Check out a named branch, then start the run again.`, + } }); + }); + + it('starts a run from a detached HEAD when no step delivers', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: fix + workspace: new + prompt: Fix the bug. +`) }, 'ship', { commit }); + + expect(started).toMatchObject({ data: { status: 'running' } }); + }); +}); diff --git a/src/__tests__/workflow-parse.test.ts b/src/__tests__/workflow-parse.test.ts index 39e6a031e..ee40b9362 100644 --- a/src/__tests__/workflow-parse.test.ts +++ b/src/__tests__/workflow-parse.test.ts @@ -427,6 +427,36 @@ steps: loop: { back_to: review, max_iterations: 3 } `; +describe('deliver', () => { + const FIX = `schemaVersion: v1alpha1 +name: build +description: Fix a bug. +steps: + - id: fix +`; + + it('parses deliver: pr on a step that writes a branch', () => { + const result = parse(`${FIX} command: node fix.js + workspace: new + deliver: pr +`); + expect(result.ok && result.definition.steps[0]).toMatchObject({ deliver: 'pr' }); + }); + + it('rejects a delivery other than pr, and deliver: pr on a step that can only read', () => { + expect(rejected(`${FIX} prompt: Fix it. + deliver: merge +`)).toEqual([{ message: `${FILE}:7: field "steps[0].deliver": step "fix" deliver must be pr; TeamAI opens a pull request and never merges`, code: 'bad_delivery' }]); + expect(rejected(`${FIX} prompt: Fix it. + workspace: read-only + deliver: pr +`)).toEqual([{ message: `${FILE}:8: field "steps[0].deliver": step "fix" has deliver: pr but runs read-only; set workspace: new so it has a branch to deliver`, code: 'bad_delivery' }]); + expect(rejected(`${FIX} command: node fix.js + deliver: pr +`)).toEqual([{ message: `${FILE}:7: field "steps[0].deliver": step "fix" has deliver: pr but runs read-only, as a command without workspace does; set workspace: new so it has a branch to deliver`, code: 'bad_delivery' }]); + }); +}); + describe('step loops', () => { it('parses a loop back to a step the owner needs, with an optional while', () => { const result = parse(`${REVIEW_AND_FIX} - id: decide diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index 2231c56b8..876eab593 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -218,6 +218,8 @@ describe('#1083 workflow run records', () => { 'judge_model_unavailable', // Refusals of a teamai score or teamai workflow command line before it reads any run or judge work. 'invalid_arguments', 'invalid_pass', 'not_initialized', 'unsupported_driver', 'invalid_tools', + // Refusals of teamai workflow start for a run whose pull requests would have no branch to target. + 'no_base_branch', // Refusals of teamai score --run, answered to the member who asked for a pass to be judged. 'scoring_unavailable', 'pass_not_found', 'scorer_not_found', 'entry_leased', // Refusals of teamai score --session, answered to the member who asked for a session to be judged. @@ -234,7 +236,7 @@ describe('#1083 workflow run records', () => { ...[...text.matchAll(/(? match[1] ?? []), ])); // An `error(…)` whose code is not a literal passes on a code written as a literal elsewhere; each one is named here. - const passedOn = ['session-flow/scoring.ts: result.code', 'workflow/engine.ts: tree.failed.code', 'workflow/scoring.ts: built.code', 'workflow/scoring.ts: result.code']; + const passedOn = ['session-flow/scoring.ts: result.code', 'workflow/engine.ts: tree.failed.code', 'workflow/engine.ts: undeliverable.code', 'workflow/scoring.ts: built.code', 'workflow/scoring.ts: result.code']; const unnamed = files.flatMap(({ file, text }) => [...text.matchAll(/(? `${file}: ${match[1]}`)).sort(); const failedWith = (code: string): RunState => runState({ status: 'failed', diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 2d5cbd054..304173ff3 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -24,7 +24,7 @@ import { import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { branchName, planWorkspace, plannedBranch, sessionOf, type StepChanges, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { branchName, deliveryProblem, planWorkspace, plannedBranch, sessionOf, type StepChanges, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; import { deliverPending, validateReportedPr } from './delivery.js'; import { generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { pinScorers } from '../scorers/cache.js'; @@ -818,6 +818,8 @@ export async function startRun( if (mismatch) return error('invalid_inputs', mismatch); const tree = await loadWorkflowTree(config, cached); if ('failed' in tree) return error(tree.failed.code, tree.failed.message); + const undeliverable = deliveryProblem(cached.definition, tree.workflows, options.base); + if (undeliverable) return error(undeliverable.code, undeliverable.message); const scorers = await pinScorers(config, listedScorers([cached.definition, ...Object.values(tree.workflows)])); const now = new Date().toISOString(); const run: RunState = { diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 6b60a7e01..bbf3536b0 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -499,10 +499,15 @@ function parseBody( const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); const deliver = scalarString(deliverPair?.value)?.trim(); - // A command without workspace is always read-only, including inside a writable workflow. - if (deliverPair && (deliver !== 'pr' || workspace === 'read-only' || (commandPair && workspace === undefined))) { + // Only what no enclosing step can change is checked per file. A prompt without workspace may continue the copy of + // the step that runs its workflow, so workflow start checks it on the whole tree. + const undeliverable = deliver !== 'pr' ? `${label} deliver must be pr; TeamAI opens a pull request and never merges` + : workspace === 'read-only' ? `${label} has deliver: pr but runs read-only; set workspace: new so it has a branch to deliver` + : commandPair && workspace === undefined ? `${label} has deliver: pr but runs read-only, as a command without workspace does; set workspace: new so it has a branch to deliver` + : undefined; + if (deliverPair && undeliverable) { diagnostics.push({ - ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, 'deliver must be pr on a writable branch'), + ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, undeliverable), code: 'bad_delivery', }); } diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 4b1b32160..a02c33aa1 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -6,7 +6,7 @@ import { itemsAfter, itemsOf } from './fan-out.js'; import { localId, statesIn, stepDefinitions, type Scope } from './scope.js'; import { parseStepPath } from './step-path.js'; import { passOf, type StepState } from './step-state.js'; -import type { DriverIdentity, RunState, WorkflowFailure, WorkflowStep } from './types.js'; +import type { DriverIdentity, RunState, WorkflowDefinition, WorkflowFailure, WorkflowStep } from './types.js'; export type WorkspaceMode = 'read-only' | 'new' | 'continue'; @@ -82,6 +82,41 @@ export function resolveMode(step: WorkflowStep, ancestors: WorkflowStep[]): Work return 'read-only'; } +/** + * Why a run of `definition` cannot deliver what its steps declare, or undefined when it can. A step in a sub-workflow + * continues the copy of the step that runs it, so this reads the whole tree a run executes, which one file cannot. + * `base` is the checkout the run starts from: a pull request targets a branch, and a detached HEAD has none. + */ +export function deliveryProblem( + definition: WorkflowDefinition, workflows: Record, base: RunState['base'], +): WorkflowFailure | undefined { + let delivering: string | undefined; + const visit = (steps: WorkflowStep[], ancestors: WorkflowStep[], prefix: string): WorkflowFailure | undefined => { + for (const step of steps) { + const path = `${prefix}${step.id}`; + if (step.deliver === 'pr') { + delivering ??= path; + if (step.workflow !== undefined && step.workspace !== 'new' && !step.forEach) { + return { code: 'bad_delivery', message: `Step "${path}" delivers its workflow, so it needs workspace: new to own the branch it delivers.` }; + } + if (step.workflow === undefined && resolveMode(step, ancestors) === 'read-only') { + return { code: 'bad_delivery', message: `Step "${path}" has deliver: pr but runs read-only. Set workspace: new on it, or on the workflow step that runs it.` }; + } + } + const child = typeof step.workflow === 'string' ? workflows[step.workflow] : step.workflow; + const problem = child && visit(child.steps, [...ancestors, step], `${path}.`); + if (problem) return problem; + } + return undefined; + }; + const problem = visit(definition.steps, [], ''); + if (problem || delivering === undefined || !base || base.ref !== undefined) return problem; + return { + code: 'no_base_branch', + message: `Step "${delivering}" delivers a pull request, but the run would start from a detached HEAD at ${base.commit}, so no branch is there for the pull request to target. Check out a named branch, then start the run again.`, + }; +} + /** How step instance `path` runs, or undefined when the run's definitions have no such step. */ export function stepMode(run: StepTree, path: string): WorkspaceMode | undefined { const definition = stepDefinitions(run)(path); From d904910a391a0196a689fc2bc91dd52a460bc38d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 11:23:22 +0200 Subject: [PATCH 06/77] feat(workflow): deliver a driver-reported pull request from workflow next MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step with deliver: pr now settles its agent's report into delivering, and the next call of any driver delivers it before it answers. The run lock is held only to pick a delivery and to apply its outcome, never across a provider, and a pid-owned delivery lock per run keeps two processes of one machine from delivering the same step twice. A process that finds the lock held answers wait with the steps still delivering, and the local driver polls rather than giving up as busy. A delivered pull request is recorded once on the branch it was opened from, under copies[branch].pr, so a retry or a second pass cannot open another, and the end record lists those as prs in step and pass order. A record merged for a run that ends again without prs drops the old list. The report hands back cleanup: delivery reads the pushed branch, so the worktree is no longer kept for it. Until TeamAI opens pull requests itself, a step with no reported URL fails with delivery_failed naming the branch that keeps the work and the retry command. The deadline the contributor computed from timeout fields main does not have is removed with the delivery module that used it. Red: "delivers a driver-reported URL without opening a PR, and leases the dependent in the same next" failed with "expected { data: { accepted: true, …(2) } } to deeply equal { data: { accepted: true, …(3) } }": the report delivered inline, failed the step at revision 3, and held back cleanup. "keeps the local driver polling while another process delivers" failed with "expected { error: { code: 'busy', …(1) } }", and the two prs record tests failed against the previous records module. --- src/__tests__/workflow-delivery.test.ts | 300 ++++++++-------------- src/__tests__/workflow-records.test.ts | 30 +++ src/__tests__/workflow-step-state.test.ts | 1 + src/workflow/conformance.ts | 17 +- src/workflow/delivery.ts | 198 -------------- src/workflow/engine.ts | 205 ++++++++------- src/workflow/local-driver.ts | 13 +- src/workflow/pr-delivery.ts | 51 ++++ src/workflow/pr-url.ts | 46 +++- src/workflow/records.ts | 27 +- src/workflow/run-controls.ts | 9 +- src/workflow/snapshot.ts | 15 +- src/workflow/step-state.ts | 53 ++-- src/workflow/store.ts | 5 + src/workflow/types.ts | 13 +- src/workflow/workspace.ts | 23 +- 16 files changed, 462 insertions(+), 544 deletions(-) delete mode 100644 src/workflow/delivery.ts create mode 100644 src/workflow/pr-delivery.ts diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 36bb9c2b2..6f514f997 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -4,24 +4,22 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; +import { z } from 'zod'; import { parseWorkflow } from '../workflow/parse.js'; -import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; +import { leaseNext, parseReport, reportAttempt, startRun, withIgnored } from '../workflow/engine.js'; import { sanitizePrUrl } from '../workflow/pr-url.js'; -import { deliveryTarget, protectedDeliveryPaths, validateReportedPr } from '../workflow/delivery.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; -import { readRun, updateRun } from '../workflow/store.js'; - -async function writeRun(config: LocalConfig, run: RunState): Promise { - await updateRun(config, run.id, async () => ({ value: undefined, write: run })); -} -import { runLocalDriver } from '../workflow/local-driver.js'; +import { deliveryLockPath, readRun } from '../workflow/store.js'; +import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; +import { acquireLock, releaseLock } from '../update.js'; import { GitHubProvider } from '../providers/github/index.js'; import { buildEndRecord } from '../workflow/records.js'; -import { cancelRun, retryStep } from '../workflow/run-controls.js'; -import { settled, stopped } from '../workflow/step-state.js'; +import { cancelRun } from '../workflow/run-controls.js'; import type { LocalConfig } from '../types.js'; import type { RunState } from '../workflow/types.js'; +_setDriverPollMs(20); + const dirs: string[] = []; afterEach(() => { vi.restoreAllMocks(); @@ -64,17 +62,38 @@ async function state(config: LocalConfig, runId: string): Promise { return run; } -const command = `node -e "require('fs').writeFileSync('change.txt', 'change')"`; -const driver = { kind: 'local', machine: 'fixture', workspace: 'fixture' }; +const LeaseSchema = z.object({ + step: z.string(), attempt: z.string(), revision: z.number(), + workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), +}); +type Lease = z.infer; +const remote = { kind: 'remote', machine: 'paseo', workspace: 'session-1' }; +const writable = { workspaceModes: ['read-only', 'new', 'continue'] as Array<'read-only' | 'new' | 'continue'> }; +const head = 'c'.repeat(40); + +/** Takes the next lease as a driver that speaks only CLI JSON does. */ +async function take(config: LocalConfig, runId: string): Promise { + const next = await leaseNext(config, runId, remote, writable); + if (!('data' in next)) throw new Error(JSON.stringify(next)); + return LeaseSchema.parse(next.data.lease); +} -function provider() { - const lookup = vi.spyOn(GitHubProvider.prototype, 'findOpenPullRequest').mockResolvedValue(undefined); - const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest').mockResolvedValue(true); - let number = 0; - const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockImplementation(async () => `https://github.com/fixture/app/pull/${++number}`); - return { lookup, validate, create }; +/** Reports through the wire payload `teamai workflow report --json` reads. */ +async function report(config: LocalConfig, runId: string, body: Record) { + const parsed = parseReport({ driver: remote, ...body }); + if (typeof parsed === 'string') throw new Error(parsed); + return withIgnored(await reportAttempt(config, runId, parsed.report), parsed.ignored); } +/** Reports a lease's success with one commit on its branch, and `changes` fields the driver adds. */ +async function succeed(config: LocalConfig, runId: string, lease: Lease, changes: Record = {}) { + const base = lease.workspace?.base.commit; + return report(config, runId, { attempt: lease.attempt, revision: lease.revision, outcome: 'success', + ...(lease.workspace ? { changes: { branch: lease.workspace.branch, base, from: base, to: head, commits: [head], ...changes } } : {}) }); +} + +const FIX_THEN_REVIEW = ` - id: fix\n workspace: new\n command: node fix.js\n deliver: pr\n - id: review\n needs: [fix]\n command: echo review\n`; + describe('workflow PR delivery contract', () => { it('parses delivery on a writable step and rejects unsupported delivery', () => { const yaml = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: write\n command: node write.js\n workspace: new\n deliver: pr\n'; @@ -87,11 +106,11 @@ describe('workflow PR delivery contract', () => { expect(parseWorkflow('ship.yaml', inherited.replace('prompt: Edit the code.', 'command: echo edit')).ok).toBe(false); }); - it('keeps the URL for validation and names unread nested fields without values', () => { + it('reads the reported URL from changes.pr_url and names unread nested fields without values', () => { const parsed = parseReport({ attempt: 'a1', revision: 1, outcome: 'success', driver: { kind: 'remote', machine: 'host', workspace: 'work' }, changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', private: 'secret' } }); - expect(parsed).toMatchObject({ report: { changes: { pr_url: 'https://review.example/1' } }, ignored: ['changes.private'] }); + expect(parsed).toMatchObject({ report: { prUrl: 'https://review.example/1' }, ignored: ['changes.private'] }); expect(JSON.stringify(parsed)).not.toContain('secret'); }); @@ -117,196 +136,85 @@ describe('workflow PR delivery contract', () => { }); }); -describe('durable PR delivery', () => { - it('pushes a branch and opens a PR against the recorded base without merging', async () => { - const api = provider(); - const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - const run = await state(config, runId); - expect(run.steps[0]).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/1' } }); - expect(api.create).toHaveBeenCalledWith(expect.objectContaining({ repo: 'fixture/app', source: `teamai/${runId}/write/p1`, target: 'main' })); - expect(git(root, ['--git-dir=remote.git', 'show', `refs/heads/teamai/${runId}/write/p1:change.txt`])).toBe('change'); - expect(git(cwd, ['show', 'main:base.txt'])).toBe('base'); - expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); - }); - - it('keeps pushed work after missing credentials and retries delivery without rerunning the agent', async () => { - const api = provider(); - api.lookup.mockRejectedValue(new Error('Set GITHUB_TOKEN before resuming.')); - const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); - let run = await state(config, runId); - const branch = `teamai/${runId}/write/p1`; - expect(run.steps[0]).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(branch) } }); - expect(git(root, ['--git-dir=remote.git', 'show', `${branch}:change.txt`])).toBe('change'); - const attempts = run.settledCount; - api.lookup.mockResolvedValue('https://github.com/fixture/app/pull/88'); - expect(await retryStep(config, runId, 'write')).toMatchObject({ data: { reopened: true } }); - run = await state(config, runId); - expect(run.steps[0]?.status).toBe('delivering'); - expect(settled(run.steps[0])).toBe(false); - expect(stopped(run.steps[0]!)).toBe(false); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - run = await state(config, runId); - expect(run.settledCount).toBe(attempts); - expect(api.create).not.toHaveBeenCalled(); - expect(run.steps[0]).toMatchObject({ changes: { branch, pr_url: 'https://github.com/fixture/app/pull/88' } }); - }); - - it('checks protected changes against the run base before any provider request', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/evil.yml','bad')"\n workspace: new\n deliver: pr\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); - expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/evil.yml') } }); - expect(api.create).not.toHaveBeenCalled(); expect(api.lookup).not.toHaveBeenCalled(); - expect(protectedDeliveryPaths(['src/a.ts', '.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit'])).toEqual(['.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit']); - }); - - it('delivers a continued composite branch once and follows inherited needs', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: change\n needs: [first]\n workspace: new\n deliver: pr\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').writeFileSync('second.txt','second')"\n - id: review\n needs: [write]\n command: cat second.txt\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - expect(api.create).toHaveBeenCalledTimes(2); - expect(api.create.mock.calls[1]?.[0]).toMatchObject({ source: `teamai/${runId}/change/p1`, target: `teamai/${runId}/first/p1` }); - expect((await state(config, runId)).steps.find((step) => step.id === 'change')).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/2' } }); - }); - - it('delivers each after-item against its actual origin and publishes PRs in item order', async () => { - const api = provider(); - const { config, cwd, root, runId } = await fixture(` - id: packages\n for_each:\n items: inputs.packages\n workspace: new\n deliver: pr\n command: node -e "require('fs').writeFileSync(process.env.PACKAGE+'.txt',process.env.PACKAGE)"\n env:\n PACKAGE: "{{ item.name }}"\n`, { packages: [{ name: 'one', id: 'one', after: [] }, { name: 'two', id: 'two', after: ['one'] }, { name: 'three', id: 'three', after: ['two'] }] }); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - expect(api.create.mock.calls.map(([opts]) => opts.target)).toEqual(['main', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`]); - expect(git(root, ['--git-dir=remote.git', 'diff', '--name-only', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`])).toBe('two.txt'); - expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); - }); - - it('accepts sanitized remote links, drops unsafe/mismatched ones without exposing their values', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); - const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'], base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) } }); - const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; - if (!lease) throw new Error(JSON.stringify(next)); - const changes = { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: 'https://user:token@github.com/fixture/app/pull/1' }; - const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes }); - expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); - expect(JSON.stringify([accepted, await state(config, runId)])).not.toContain('token'); - expect(api.validate).not.toHaveBeenCalled(); - }); - - it.each([ - { url: 'https://review.example/changes/1?session=secret#note', valid: undefined, expected: 'https://review.example/changes/1', reason: undefined }, - { url: 'https://github.com/fixture/app/pull/1', valid: false, expected: undefined, reason: 'repository or branches do not match the lease' }, - ])('records only a validated driver link ($url)', async ({ url, valid, expected, reason }) => { - const api = provider(); api.validate.mockResolvedValue(valid); - const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); - const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'] }); - const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; - if (!lease) throw new Error('missing lease'); - const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes: { - branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: url, - } }); - expect(accepted).toMatchObject({ data: { accepted: true, ...(reason ? { dropped: [{ field: 'changes.pr_url', reason }] } : {}) } }); +describe('delivering a step', () => { + it('delivers a driver-reported URL without opening a PR, and leases the dependent in the same next', async () => { + const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockRejectedValue(new Error('no PR may be opened')); + const { config, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + expect(fix).toMatchObject({ step: 'fix', workspace: { branch } }); + + expect(await succeed(config, runId, fix, { pr_url: 'https://review.acme.internal/changes/4821?session=abc#c3' })) + .toEqual({ data: { accepted: true, revision: 2, step: { id: 'fix', status: 'delivering', outcome: 'success' }, cleanup: true } }); + + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + try { + expect(await leaseNext(config, runId, remote, writable)).toEqual({ data: { wait: { reason: 'fix is delivering', delivering: ['fix'] }, cancelled: [] } }); + } finally { + await releaseLock(held); + } + + const review = await take(config, runId); + expect(review.step).toBe('review'); const run = await state(config, runId); - expect(buildEndRecord(run, driver, 'tester').prs).toEqual(expected ? [expected] : undefined); - expect(JSON.stringify(run)).not.toContain('session='); - }); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', settledAt: 2 }); + expect(run.copies?.[branch]?.pr).toEqual({ url: 'https://review.acme.internal/changes/4821', step: 'fix', pass: 1 }); + expect(create).not.toHaveBeenCalled(); - it('can cancel a persisted delivery without treating it as settled success', async () => { - const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - const run = await state(config, runId); - run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; - await writeRun(config, run); - expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); - expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + await report(config, runId, { attempt: review.attempt, revision: review.revision, outcome: 'success' }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://review.acme.internal/changes/4821']); }); - it('walks undelivered origin branches and deduplicates URLs from passes and archives', async () => { - const { config, runId } = await fixture(` - id: write\n command: echo test\n`); - const run = await state(config, runId); - const changes = { branch: 'first', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', prRecordedAt: 2 }; - run.copies = { second: { owner: driver, origin: 'first' }, third: { owner: driver, origin: 'second' } }; - run.steps = [{ id: 'write', status: 'succeeded', outcome: 'success', settledAt: 4, changes: { ...changes, branch: 'third', prRecordedAt: 4 }, passes: [{ pass: 1, outcome: 'success', changes }] }]; - run.archive = [{ step: 'older', pass: 1, outcome: 'success', changes: { ...changes, pr_url: 'https://review.example/2', prRecordedAt: 3 } }]; - expect(deliveryTarget(run, 'third')).toBe('first'); - expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://review.example/1', 'https://review.example/2']); - if (run.steps[0]?.status === 'succeeded') run.steps[0].outputs = { pr_url: 'https://untrusted.example/no' }; - expect(buildEndRecord(run, driver, 'tester').prs).not.toContain('https://untrusted.example/no'); - }); + it('keeps the local driver polling while another process delivers, rather than giving up as busy', async () => { + const { config, cwd, runId } = await fixture(FIX_THEN_REVIEW); + await succeed(config, runId, await take(config, runId), { pr_url: 'https://review.acme.internal/changes/1' }); + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + const released = new Promise((resolve) => setTimeout(() => void releaseLock(held).then(resolve), 200)); - it('opens a separate PR per new-workspace loop pass and retains all earlier URLs', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - expect(api.create.mock.calls.map(([opts]) => opts.source)).toEqual([1, 2, 3].map((pass) => `teamai/${runId}/write/p${pass}`)); - expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + await released; }); - it('reuses one PR when a loop continues the same branch', async () => { - const api = provider(); - api.lookup.mockImplementation(async () => api.create.mock.calls.length ? 'https://github.com/fixture/app/pull/1' : undefined); - const { config, cwd, runId } = await fixture(` - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').appendFileSync('change.txt','pass\\n')"\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - expect(api.create).toHaveBeenCalledTimes(1); expect(api.lookup).toHaveBeenCalledTimes(3); - expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); - }); + it('fails a delivery that has no URL and no host to open one on, naming the branch that keeps the work', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + git(cwd, ['remote', 'set-url', 'origin', path.join(root, 'remote.git')]); + expect(await succeed(config, runId, await take(config, runId))).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); - it('passes over a skipped first need to the earlier delivered branch', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: other\n needs: []\n workspace: new\n command: node -e "require('fs').writeFileSync('other.txt','other')"\n deliver: pr\n - id: skipped\n needs: [first]\n if: steps.first.outcome == 'failed'\n command: echo skipped\n - id: last\n needs: [skipped, other]\n workspace: new\n command: node -e "require('fs').writeFileSync('last.txt','last')"\n deliver: pr\n`); - expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); - expect(api.create.mock.calls[2]?.[0]).toMatchObject({ source: `teamai/${runId}/last/p1`, target: `teamai/${runId}/first/p1` }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const branch = `teamai/${runId}/fix/p1`; + expect((await state(config, runId)).steps).toMatchObject([ + { id: 'fix', status: 'failed', delivery: { branch, head }, failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }, + { id: 'review', status: 'skipped' }, + ]); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); }); - it('rejects inherited protected changes even when the delivering step did not edit those paths', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: earlier\n workspace: new\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/ci.yml','bad')"\n - id: later\n needs: [earlier]\n workspace: new\n command: ${command}\n deliver: pr\n`); - expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); - expect((await state(config, runId)).steps[1]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/ci.yml') } }); - expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); - }); + it('cancels a delivering step: it delivers nothing and its dependents do not run', async () => { + const { config, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { pr_url: 'https://review.acme.internal/changes/1' }); - it('allows cancellation during provider I/O and retains the resulting PR receipt without reviving the step', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - api.create.mockImplementation(async () => { - expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); - return 'https://github.com/fixture/app/pull/99'; - }); - expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect(await cancelRun(config, runId, 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'], attempts: [] }, run: { status: 'failed' } } }); const run = await state(config, runId); - expect(run.steps[0]).toMatchObject({ status: 'cancelled', changes: { pr_url: 'https://github.com/fixture/app/pull/99' } }); - expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/99']); - }); - it('does not create a PR when cancellation completed during the open-request lookup', async () => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - api.lookup.mockImplementation(async () => { - await cancelRun(config, runId, 'write'); - return undefined; - }); - expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); - expect(api.create).not.toHaveBeenCalled(); - expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + expect(run.steps).toMatchObject([{ id: 'fix', status: 'cancelled', attempt: fix.attempt }, { id: 'review', status: 'skipped' }]); + expect(run.copies?.[`teamai/${runId}/fix/p1`]?.pr).toBeUndefined(); }); - it('keeps an execution deadline across delivery and makes no provider call once it expired', async () => { - const api = provider(); - const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); - const run = await state(config, runId); - run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, deliveryDeadline: Date.now() - 1, - changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; - await writeRun(config, run); - expect(await leaseNext(config, runId, driver)).toMatchObject({ data: { done: { outcome: 'failed' } } }); - expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { code: 'timeout' } }); - expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); - }); + it.todo('delivers a workflow step with workspace: new once, against the delivered branch of the step it needs'); + it.todo('delivers each for_each item against the delivered branch of the item it waited for, and lists the PRs in item order'); + it.todo('targets the nearest delivered branch, passing over a need that was skipped'); +}); - it.each(['https://oauth2:fixture-token@github.com/fixture/app.git', 'ssh://git@github.com:22/fixture/app.git', 'git@github.com:fixture/app.git'])('recognizes hosting metadata through %s without passing transport credentials', async (remote) => { - const api = provider(); - const { config, cwd, runId } = await fixture(` - id: write\n command: echo test\n`); - git(cwd, ['remote', 'set-url', 'origin', remote]); - expect(await validateReportedPr(await state(config, runId), { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ url: 'https://github.com/fixture/app/pull/1' }); - expect(api.validate).toHaveBeenCalledWith('https://github.com/fixture/app/pull/1', expect.objectContaining({ repo: 'fixture/app' })); - expect(JSON.stringify(api.validate.mock.calls)).not.toContain('fixture-token'); +describe('reported URLs from a driver', () => { + it('drops a URL with credentials without failing the report, and never stores or echoes it', async () => { + const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest'); + const { config, runId } = await fixture(` - id: write\n command: node write.js\n workspace: new\n`); + const accepted = await succeed(config, runId, await take(config, runId), { pr_url: 'https://user:token@github.com/fixture/app/pull/1' }); + expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); + expect(JSON.stringify([accepted, await state(config, runId)])).not.toContain('token'); + expect(validate).not.toHaveBeenCalled(); }); }); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index 876eab593..5647e802f 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -23,6 +23,7 @@ import { } from '../workflow/records.js'; import { acquireLock, releaseLock } from '../update.js'; import { generateRunId, isRunId, runLockPath } from '../workflow/store.js'; +import { PrUrlSchema } from '../workflow/pr-url.js'; import type { DriverIdentity, RunState, WorkflowDefinition, WorkflowDiagnostic } from '../workflow/types.js'; const dirs: string[] = []; @@ -396,6 +397,35 @@ describe('#1083 workflow run records', () => { expect(remerged.endedAt).toBe(end.endedAt); }); + it('lists the pull requests delivered on the run\'s branches once each, in step and pass order, and never a step output', () => { + const pr = (n: number): string => `https://github.com/acme/app/pull/${n}`; + const recorded = (n: number, step: string, pass: number) => ({ pr: { url: PrUrlSchema.parse(pr(n)), step, pass } }); + const run = runState({ + steps: [ + { id: 'one', status: 'succeeded', attempt: 'a1', settledAt: 2, outcome: 'success', outputs: { pr_url: 'https://untrusted.example/1' } }, + { id: 'two', status: 'succeeded', attempt: 'a2', settledAt: 3, outcome: 'success' }, + ], + copies: { + 'teamai/r1234567890abcdef/two/p2': recorded(4, 'two', 2), + 'teamai/r1234567890abcdef/two/p1': recorded(3, 'two', 1), + 'teamai/r1234567890abcdef/one/p1': recorded(2, 'one', 1), + 'teamai/r1234567890abcdef/loop/p1': recorded(1, 'loop.write', 1), + 'teamai/r1234567890abcdef/other/p1': { commit: 'c'.repeat(40) }, + }, + }); + + expect(buildEndRecord(run, driver, 'tester').prs).toEqual([pr(1), pr(2), pr(3), pr(4)]); + expect(buildEndRecord(runState(), driver, 'tester')).not.toHaveProperty('prs'); + }); + + it('drops the prs of an earlier end when a retried run ends again without them', () => { + const delivered = { ...buildEndRecord(runState(), driver, 'tester'), prs: ['https://github.com/acme/app/pull/1'] }; + const again = buildEndRecord(runState(), driver, 'tester'); + + expect(mergeRecord(delivered, again)).not.toHaveProperty('prs'); + expect(mergeRecord(again, delivered).prs).toEqual(['https://github.com/acme/app/pull/1']); + }); + it('queues records durably and keeps them across failure', async () => { const { config } = testConfig(); const record = buildStartRecord(runState({ status: 'running' }), driver, 'tester'); diff --git a/src/__tests__/workflow-step-state.test.ts b/src/__tests__/workflow-step-state.test.ts index be267d918..4d054f912 100644 --- a/src/__tests__/workflow-step-state.test.ts +++ b/src/__tests__/workflow-step-state.test.ts @@ -11,6 +11,7 @@ const step = (status: StepState['status']): StepState => StepStateSchema.parse({ id: 'work', status, ...(status === 'leased' ? { attempt: 'a1', lease: { attempt: 'a1', revision: 1, driver: { kind: 'local', machine: 'm', workspace: 'w' } } } : {}), ...(status === 'cancelled' ? { failure: { code: 'cancelled', message: 'Cancelled' } } : {}), + ...(status === 'delivering' ? { delivery: { branch: 'teamai/r1/work/p1', head: 'c'.repeat(40) } } : {}), }); describe('step statuses', () => { diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index a56819c80..5424e30a7 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -65,6 +65,16 @@ steps: command: echo work `; +const REPORTED_PR = `schemaVersion: v1alpha1 +name: reported-pr +description: A remote driver opens the pull request of the branch it worked on and reports it. +steps: + - id: work + workspace: new + command: echo work + deliver: pr +`; + const FAILED_DEPENDENCY = `schemaVersion: v1alpha1 name: build description: A step whose dependency failed is skipped. @@ -252,13 +262,14 @@ steps: export const workflowConformanceScenarios: ConformanceScenario[] = [ { - name: 'a driver reports a sanitized review URL', workflowName: 'review-link', yaml: REVIEW_LINK, + name: 'a driver reports the pull request it opened, which delivers the step without opening another', + workflowName: 'reported-pr', yaml: REPORTED_PR, operations: [ { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, - { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' } }, expect: { data: { accepted: true, ignored: ['changes.unknown'] } } }, + { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' } }, expect: { data: { accepted: true, step: { id: 'work', status: 'delivering', outcome: 'success' }, ignored: ['changes.unknown'] } } }, { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, ], - expectState: { steps: [{ id: 'work', changes: { pr_url: 'https://review.example/changes/1' } }] }, + expectState: { steps: [{ id: 'work', status: 'succeeded', delivery: { reported: 'https://review.example/changes/1' } }] }, }, { name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, diff --git a/src/workflow/delivery.ts b/src/workflow/delivery.ts deleted file mode 100644 index b24d9ecbc..000000000 --- a/src/workflow/delivery.ts +++ /dev/null @@ -1,198 +0,0 @@ -/** PR delivery is resumable I/O, separate from the agent lease and never under the run lock. */ -import path from 'node:path'; -import type { LocalConfig } from '../types.js'; -import type { GitProvider, PrCreateOptions } from '../providers/types.js'; -import { getProvider, getProviderFromUrl } from '../providers/registry.js'; -import { acquireLock, releaseLock } from '../update.js'; -import { fail, succeed, type StepState } from './step-state.js'; -import { readRun, runLockPath } from './store.js'; -import { withRun, ok } from './engine.js'; -import { sanitizePrUrl } from './pr-url.js'; -import { AGENT_PATHS, git } from './worktree.js'; -import type { StepChanges } from './workspace.js'; -import type { RunState } from './types.js'; -import { publishRunRecords, queueRunRecord } from './records.js'; - -/** Changes in earlier passes remain candidates for a stacked target. */ -function allChanges(run: RunState): StepChanges[] { - return [ - ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), - ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), - ]; -} - -/** Follows the actual copy origin, including after-item precedence, through branches that did not deliver. */ -export function deliveryTarget(run: RunState, branch: string): string | undefined { - const changes = allChanges(run); - const visited = new Set([branch]); - let origin = run.copies?.[branch]?.origin; - while (origin && !visited.has(origin)) { - visited.add(origin); - if (changes.some((item) => item.branch === origin && item.pr_url)) return origin; - origin = run.copies?.[origin]?.origin; - } - return run.base?.ref; -} - -const CI_PATHS = ['.github/workflows', '.github/actions', '.gitlab-ci.yml', '.gitlab', '.coding-ci.yaml', '.cnb', '.circleci', '.buildkite', '.husky', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml']; - -export function protectedDeliveryPaths(files: string[]): string[] { - const roots = [...AGENT_PATHS, ...CI_PATHS].map((root) => root.toLowerCase()); - return files.filter((file) => { - const lower = file.toLowerCase(); - return /(^|\/)(agents|claude)\.md$/.test(lower) - || roots.some((root) => lower === root || lower.startsWith(`${root}/`)); - }); -} - -async function providerFor(run: RunState): Promise<{ provider: GitProvider; opts: PrCreateOptions }> { - if (!run.checkout) throw new Error('The run has no product checkout. Resume from a checkout that can access the delivered branch.'); - // Read the configured URL: `remote get-url` expands insteadOf transport rewrites and can hide the hosting provider. - const remote = await git(run.checkout, ['config', '--get', 'remote.origin.url']); - if (remote.code !== 0) throw new Error('The product checkout has no origin remote. Configure origin before resuming.'); - const url = remote.stdout.trim(); - let hostingUrl = url; - if (/^(https?|ssh):\/\//.test(url)) { - const parsed = new URL(url); - parsed.username = ''; parsed.password = ''; parsed.search = ''; parsed.hash = ''; - hostingUrl = parsed.protocol === 'ssh:' ? `https://${parsed.hostname}${parsed.pathname}` : parsed.toString(); - } else { - const scp = /^[^@\s]+@([^:\s]+):(.+)$/.exec(url); - if (scp) hostingUrl = `https://${scp[1]}/${scp[2]}`; - } - // Filesystem remotes are transport-only too; the registry's bare owner/repo fallback is for user input. - const provider = /^(https?:\/\/|ssh:\/\/|[^@\s]+@[^:\s]+:)/.test(url) ? getProviderFromUrl(hostingUrl) : getProvider('git'); - const repo = provider.name === 'git' ? hostingUrl : (() => { - const parsed = provider.parseRepoInput(hostingUrl); - return `${parsed.owner}/${parsed.repo}`; - })(); - return { provider, opts: { repo, source: '', target: '', title: '', cwd: run.checkout, spawnTimeoutMs: 30_000 } }; -} - -/** Report validation never echoes a rejected value, including a provider's error response. */ -export async function validateReportedPr(run: RunState, changes: StepChanges): Promise<{ url?: string; reason?: string }> { - const clean = sanitizePrUrl(changes.pr_url); - if (!clean.url) return clean; - if (!run.checkout) return clean; - try { - const { provider, opts } = await providerFor(run); - const target = deliveryTarget(run, changes.branch); - if (!target) return { reason: 'run base has no branch' }; - const valid = await provider.validatePullRequest?.(clean.url, { ...opts, source: changes.branch, target }); - return valid === false ? { reason: 'repository or branches do not match the lease' } : clean; - } catch { - return { reason: 'provider metadata could not be validated' }; - } -} - -async function createDelivery(config: LocalConfig, run: RunState, state: Extract): Promise { - const remaining = (): number => { - const ms = state.deliveryDeadline === undefined ? 30_000 : Math.min(30_000, state.deliveryDeadline - Date.now()); - if (ms <= 0) throw new Error('The execution deadline expired before delivery completed.'); - return ms; - }; - remaining(); - const changes = state.changes; - if (!changes || !run.base || !run.checkout) throw new Error('Delivery needs a writable branch, a run base, and its product checkout.'); - const target = deliveryTarget(run, changes.branch); - if (!target) throw new Error('Run base has no branch. Start the run from a named branch (CI checkout ref: main).'); - if (![changes.to, run.base.commit].every((commit) => /^[0-9a-f]{40,64}$/i.test(commit))) throw new Error('Delivery requires full commit object ids.'); - // Do not inspect a later branch head or trust a driver's claimed list of commits. - let head = await git(run.checkout, ['rev-parse', '--verify', `refs/heads/${changes.branch}^{commit}`]); - if (head.code !== 0 || head.stdout.trim() !== changes.to) { - const fetched = await git(run.checkout, ['fetch', '--no-tags', 'origin', `refs/heads/${changes.branch}`]); - head = await git(run.checkout, ['rev-parse', '--verify', 'FETCH_HEAD^{commit}']); - if (fetched.code !== 0 || head.code !== 0 || head.stdout.trim() !== changes.to) throw new Error('The reported branch head is not available. Push the reported commit before resuming.'); - } - const diff = await git(run.checkout, ['diff', '--no-renames', '--name-only', '-z', run.base.commit, changes.to, '--']); - if (diff.code !== 0) throw new Error('Cannot inspect the delivered diff against the run base. Fetch the base commit before resuming.'); - const protectedPaths = protectedDeliveryPaths(diff.stdout.split('\0').filter(Boolean)); - if (protectedPaths.length) throw new Error(`Protected paths changed against the run base: ${protectedPaths.join(', ')}`); - const { provider, opts } = await providerFor(run); - opts.spawnTimeoutMs = remaining(); - if (changes.pr_url) return changes.pr_url; - const request: PrCreateOptions = { ...opts, source: changes.branch, target, - title: `${run.name}: ${state.id}`, description: `Workflow ${run.name}, run ${run.id}, step ${state.id}.\n\nBase: ${run.base.commit}\n\nCreated for human review. TeamAI never merges this request.` }; - const existing = await provider.findOpenPullRequest?.(request); - let url = existing; - if (!url) { - const latest = await readRun(config, run.id); - const active = typeof latest !== 'string' && latest.steps.find((step) => step.id === state.id); - if (typeof latest === 'string' || latest.status !== 'running' || !active || active.status !== 'delivering' - || active.pass !== state.pass || active.attempt !== state.attempt || active.settledAt !== state.settledAt) { - throw new Error('Delivery was cancelled or replaced during PR lookup; no PR was created.'); - } - request.spawnTimeoutMs = remaining(); - try { url = await provider.createPullRequest(request); } - catch (error) { - // A provider's response body or CLI output may contain credentials. Keep its established transport-only - // fallback, but never persist arbitrary creation errors from authenticated providers. - if (provider.name === 'git') throw error; - throw new Error(`The ${provider.name} provider could not create the PR. Check its configured credentials, repository/branch access and push permissions, then retry delivery.`); - } - } - const clean = sanitizePrUrl(url); - if (!clean.url) throw new Error('Provider returned an unsafe PR URL. The branch is preserved; inspect the provider before retrying.'); - return clean.url; -} - -/** Drains durable delivery work once. A competing delivery process leaves it for the next poll. */ -export async function deliverPending(config: LocalConfig, runId: string): Promise { - const loaded = await readRun(config, runId); - if (typeof loaded === 'string' || loaded.status !== 'running' || !loaded.steps.some((step) => step.status === 'delivering')) return false; - const lock = path.join(path.dirname(runLockPath(config, runId)), `${runId}.delivery.lock`); - if (!await acquireLock(lock)) return false; - let changed = false; - let queued = false; - try { - for (;;) { - const run = await readRun(config, runId); - if (typeof run === 'string' || run.status !== 'running') break; - const state = run.steps.find((step): step is Extract => step.status === 'delivering'); - if (!state) break; - let url: string | undefined; - let failure: string | undefined; - try { url = await createDelivery(config, run, state); } - catch (error) { failure = error instanceof Error ? error.message : 'PR delivery failed'; } - const applied = await withRun(config, runId, async (current) => { - const step = current.steps.find((item) => item.id === state.id); - // Cancel/retry may have invalidated this delivery while the provider was running. - if (current.status !== 'running' || step?.status !== 'delivering' || step.pass !== state.pass || step.settledAt !== state.settledAt) { - // An in-flight provider request cannot be revoked. Keep its safe receipt on the matching completed - // attempt/pass, without resurrecting cancelled work, so the end record can still account for the PR. - if (!url || !state.changes) return { save: false, envelope: ok({}) }; - const candidates = [...(step && 'changes' in step ? [step] : []), ...(step?.passes ?? []), - ...(current.archive ?? []).filter((pass) => pass.step === state.id)]; - const receipt = candidates.find((pass) => pass.changes?.branch === state.changes!.branch && pass.changes.to === state.changes!.to - && pass.attempt === state.attempt && (pass.pass ?? 1) === (state.pass ?? 1)); - if (!receipt?.changes) return { save: false, envelope: ok({}) }; - current.revision += 1; - receipt.changes.pr_url = url; - receipt.changes.prRecordedAt ??= current.revision; - if (current.status !== 'running') queued = await queueRunRecord(config, current, current.driver, 'end') || queued; - changed = true; - return { save: true, envelope: ok({}) }; - } - current.revision += 1; - const { status: _status, ...settlement } = step; - const expired = step.deliveryDeadline !== undefined && Date.now() >= step.deliveryDeadline; - if (failure || expired) { - const message = `PR delivery failed for branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved. ${failure}`; - const detail = expired ? `Execution deadline expired while delivering branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved.` : message; - fail(step, { ...settlement, settledAt: current.revision, failure: { code: expired ? 'timeout' : 'delivery_failed', message: detail }, message: detail, - ...(url && step.changes ? { changes: { ...step.changes, pr_url: url, prRecordedAt: step.changes.prRecordedAt ?? current.revision } } : {}) }); - } else { - succeed(step, { ...settlement, settledAt: current.revision, - changes: { ...step.changes!, pr_url: url!, prRecordedAt: step.changes?.prRecordedAt ?? current.revision } }); - } - changed = true; - return { save: true, envelope: ok({}) }; - }); - if ('error' in applied) break; - } - return changed; - } finally { - await releaseLock(lock); - if (queued) await publishRunRecords(config); - } -} diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 304173ff3..5ed9ab022 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -18,15 +18,17 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - ReadSchema, answerHitl, cancel, deliver, fail, inProgress, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, - type ItemResult, type LeasedStep, type StepRead, type StepState, + ReadSchema, answerHitl, cancel, delivered, deliveryFailed, fail, inProgress, lease, release, settled, skip, start, startDelivery, stopped, + succeed, waitForAnswer, type DeliveringStep, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { branchName, deliveryProblem, planWorkspace, plannedBranch, sessionOf, type StepChanges, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; -import { deliverPending, validateReportedPr } from './delivery.js'; -import { generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; +import { deliverableOf, deliveryProblem, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { deliverPr, deliveryFailure, type DeliveryJob, type DeliveryKey, type DeliveryOutcome } from './pr-delivery.js'; +import { sanitizePrUrl } from './pr-url.js'; +import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; +import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; import { isLeaseRef } from '../scorers/entry.js'; import type { @@ -36,7 +38,6 @@ import type { RunStatus, StepLease, StepOutcome, - WorkflowDefinition, WorkflowEnvelope, WorkflowFailure, WorkflowStep, @@ -66,7 +67,9 @@ export interface ReportInput { tokens?: TokenUsage; failure?: WorkflowFailure; /** The branch this attempt committed, when it ran in a workspace. */ - changes?: StepChanges; + changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; + /** `changes.pr_url` as the driver sent it: a pull request it opened itself, kept only once it is sanitized. */ + prUrl?: unknown; /** The branch and commit this attempt read, when it ran read-only in a workspace. */ read?: StepRead; agent?: ReportedAgent; @@ -539,24 +542,9 @@ export function stopWhere(run: RunState, under: (id: string) => boolean, failure /** * A step takes how what it ran settled: a workflow step, the outcome, the end, and the outputs of its workflow; a - * `for_each` step, the outcome and the results of its items. + * `for_each` step, the outcome and the results of its items. A workflow step or `for_each` item that delivers goes on + * to deliver what it wrote; a `for_each` step delivers nothing of its own, its items do. */ -function deliveryDeadline(run: RunState, state: StepState): number | undefined { - const limits: number[] = []; - const root = run.definition as WorkflowDefinition & { timeout?: number }; - if (root.timeout) limits.push(Date.parse(run.createdAt) + root.timeout); - if (state.status === 'leased' && 'deadline' in state.lease && typeof state.lease.deadline === 'number') limits.push(state.lease.deadline); - for (const parent of run.steps) { - if (parent.id !== state.id && !state.id.startsWith(`${parent.id}.`)) continue; - const found = locateStep(run, parent.id); - const scope = found && found.step.workflow !== undefined ? childScope(run, found.scope, statesIn(run, found.scope), parent, found.step) : undefined; - const bounds = [(found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout, - (scope?.definition as { timeout?: number } | undefined)?.timeout]; - for (const timeout of bounds) if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); - } - return limits.length ? Math.min(...limits) : undefined; -} - function finishStep(run: RunState, state: StepState, definition: WorkflowStep, result: WorkflowResult): void { const settlement = { settledAt: result.settledAt, @@ -564,19 +552,13 @@ function finishStep(run: RunState, state: StepState, definition: WorkflowStep, r ...(result.outputs !== undefined ? { outputs: result.outputs } : {}), ...(result.results ? { results: result.results } : {}), }; - if (result.outcome === 'succeeded' && definition.deliver === 'pr' && (!definition.forEach || isItem(state))) { - // A workflow's children may continue one enclosing copy. Deliver that branch once at its workflow boundary. - const own = branchName(run.id, state.id, state.pass ?? 1); - const branches = [...new Set(run.steps.flatMap((child) => child.id.startsWith(`${state.id}.`) && 'changes' in child && child.changes ? [child.changes.branch] : []))]; - const branch = run.copies?.[own] ? own : branches.length === 1 ? branches[0] : undefined; - const copy = branch && run.copies?.[branch]; - if (!branch || !copy || !copy.commit || !run.base) { - fail(state, { ...settlement, failure: { code: 'delivery_failed', message: 'PR delivery needs one writable branch; set workspace: new on the workflow step.' } }); - } else { - deliver(state, { ...settlement, changes: { branch, base: run.base.commit, from: run.base.commit, to: copy.commit, commits: [] } }, deliveryDeadline(run, state)); - } - } else if (result.outcome === 'succeeded') succeed(state, settlement); - else fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); + if (result.outcome !== 'succeeded') { + fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); + return; + } + const delivery = definition.deliver === 'pr' && (!definition.forEach || isItem(state)) ? deliverableOf(run, state, definition) : undefined; + if (delivery) startDelivery(state, settlement, delivery); + else succeed(state, settlement); } function childOrUnreadable(run: RunState, scope: Scope, states: StepState[], state: StepState, definition: WorkflowStep): Scope | { unreadable: string } { @@ -894,18 +876,89 @@ export interface NextOptions { claim?: boolean; } -export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { - const envelope = await leaseNextCore(config, runId, driver, options); - if ('error' in envelope || !await deliverPending(config, runId)) return envelope; - return leaseNext(config, runId, driver, options); +/** The delivery of `step`, as the drain and the outcome it applies name it. */ +function deliveryKey(step: DeliveringStep): DeliveryKey { + return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; +} + +/** The step `key` names, while it still delivers that pass of that commit. */ +function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep | undefined { + const step = run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && item.id === key.step); + if (!step) return undefined; + const current = deliveryKey(step); + return current.pass === key.pass && current.settledAt === key.settledAt && current.branch === key.branch && current.head === key.head ? step : undefined; +} + +/** + * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel + * stopped is not delivered. Undefined when nothing delivers, or when the run settles, which the locked body of `next` + * finishes. + */ +function nextDelivery(run: RunState): DeliveryJob | undefined { + if (run.status !== 'running') return undefined; + const settledRun = reconcileRun(run); + if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; + const step = run.steps.find((item): item is DeliveringStep => item.status === 'delivering'); + if (!step) return undefined; + return { key: deliveryKey(step), runId: run.id, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}) }; +} + +/** + * Applies a delivery's outcome. A pull request is recorded on its branch whatever became of the step meanwhile: one + * opened while a cancel landed cannot be taken back, so the run says it exists. The step settles only while it still + * delivers what `job` delivered, at the revision its agent's report settled it. + */ +function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome): void { + if (outcome.ok) { + const copies = run.copies ??= {}; + const copy = copies[job.key.branch] ?? {}; + copy.pr ??= { url: outcome.pr, step: job.key.step, pass: job.key.pass }; + copies[job.key.branch] = copy; + } + if (run.status === 'running') reconcileRun(run); + const step = deliveringStep(run, job.key); + if (!step) return; + if (outcome.ok) delivered(step); + else deliveryFailed(step, deliveryFailure(job, outcome.fault)); } -async function leaseNextCore(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions): Promise { +/** + * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it. The run lock is + * held to pick a delivery and to apply its outcome, never across git or a provider. While another process of this + * machine delivers, it holds the delivery lock, and this call leaves the work to it. + */ +async function drainDeliveries(config: LocalConfig, runId: string): Promise { + if (!isRunId(runId)) return; + const loaded = await readRun(config, runId); + if (typeof loaded === 'string' || loaded.status !== 'running' || !loaded.steps.some((step) => step.status === 'delivering')) return; + await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { + for (;;) { + const picked = await updateRun(config, runId, async (run) => { + if (run === 'missing') return { value: undefined }; + const before = JSON.stringify(run); + const job = nextDelivery(run); + return { value: job, ...(job && JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; + }); + if (picked.status !== 'done' || !picked.value) return; + const job = picked.value; + const outcome = await deliverPr(job); + const applied = await updateRun(config, runId, async (run) => { + if (run === 'missing') return { value: false }; + completeDelivery(run, job, outcome); + return { value: true, write: { ...run, updatedAt: new Date().toISOString() } }; + }); + if (applied.status !== 'done' || !applied.value) return; + } + }); +} + +export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { driver = { kind: driver.kind, machine: driver.machine, workspace: driver.workspace }; if (options.claim) { const claimed = await claimRun(config, runId, { checkout: driver.workspace }); if (!claimed.ok) return { error: claimed.error }; } + await drainDeliveries(config, runId); const running = new Set(options.running); // The end record is queued under the run lock, so the run stores whether it // was in the same save. It publishes AFTER withRun releases the lock (#1126 @@ -929,11 +982,12 @@ async function leaseNextCore(config: LocalConfig, runId: string, driver: DriverI queued = await finishRun(config, run, now, result, driver); return answer(doneData(run)); }; - const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => ( - answer({ wait: run.steps.some((step) => step.status === 'delivering') - ? { reason: 'PR delivery is in progress; poll next', delivery: true } - : { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) - ); + const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => { + // Steps still delivering here are being delivered by another process of this machine; a driver polls until they settle. + const delivering = run.steps.filter((step) => step.status === 'delivering').map((step) => step.id); + const reason = open ? `${open.id} is leased` : delivering.length > 0 ? `${delivering[0]} is delivering` : 'canTakeLease is false'; + return answer({ wait: { reason, ...(delivering.length > 0 ? { delivering } : {}) } }); + }; if (run.status !== 'running') { // Settled by an earlier next, which queued the end record unless usage // reporting was off; the done envelope reports that record's state. @@ -1143,10 +1197,7 @@ export function parseReport(value: unknown): ParsedReport | string { return 'changes.commits must be an array of strings'; } report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; - if (value.pr_url !== undefined) { - // Invalid types are dropped with a value-free reason by reportAttempt too. - report.changes.pr_url = value.pr_url as string; - } + if (value.pr_url !== undefined) report.prUrl = value.pr_url; } if (record.read !== undefined) { const read = ReadSchema.safeParse(record.read); @@ -1262,7 +1313,7 @@ function branchStillUsed(run: RunState, branch: string): boolean { const ahead = structuredClone(run); if ('unreadable' in reconcileRun(ahead)) return true; for (const item of ahead.steps) { - if (item.status === 'running' || settled(item)) continue; + if (item.status === 'running' || item.status === 'delivering' || settled(item)) continue; const located = locateStep(ahead, item.id); if (located && plannedBranch(ahead, located.scope, item, located.step) === branch) return true; } @@ -1327,20 +1378,10 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: return error('judge_lease', `Attempt ${report.attempt} is a judge lease, which workflow report does not settle. Submit its answer with: teamai score --lease ${report.attempt} --submit --stdin`); } const install = await installOwner(config); - const dropped: Array<{ field: string; reason: string }> = []; - if (report.changes && report.changes.pr_url !== undefined) { - report = structuredClone(report); - const run = await readRun(config, runId); - if (typeof run !== 'string') { - const result = await validateReportedPr(run, report.changes!); - if (result.url) report.changes!.pr_url = result.url; - else { - delete report.changes!.pr_url; - dropped.push({ field: 'changes.pr_url', reason: result.reason! }); - } - } - } - const envelope = await withRun(config, runId, async (run) => { + const reported = report.prUrl === undefined ? undefined : sanitizePrUrl(report.prUrl); + // A URL that is not kept is named without its value, which may hold a token, and the report still counts. + const dropped = reported?.reason === undefined ? [] : [{ field: 'changes.pr_url', reason: reported.reason }]; + return withRun(config, runId, async (run) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { return { save: false, envelope: error('repeated_report', `Attempt ${report.attempt} was already reported`) }; @@ -1401,7 +1442,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: model: report.model, session: report.session, tokens: report.tokens, - ...(report.changes ? { changes: { ...report.changes, prRecordedAt: report.changes.pr_url ? run.revision : undefined } } : {}), + ...(report.changes ? { changes: report.changes } : {}), ...(report.read ? { read: report.read } : {}), agent: settleObservation(report, defined?.agent), ...(report.used ? { used: report.used } : {}), @@ -1415,11 +1456,14 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: run.copies[report.changes.branch] = copy; } const successful = { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }; + const delivery = outcome === 'success' && defined?.deliver === 'pr' ? deliverableOf(run, step, defined, report.changes) : undefined; const settledStep = outcome === 'released' ? release(step) - : outcome === 'success' ? defined?.deliver === 'pr' ? deliver(step, successful, deliveryDeadline(run, step)) : succeed(step, successful) - : fail(step, { ...settlement, failure }); - // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. - const cleanup = outcome === 'success' && settledStep.status !== 'delivering' && contract !== undefined && !branchStillUsed(run, contract.branch); + : outcome !== 'success' ? fail(step, { ...settlement, failure }) + : delivery ? startDelivery(step, successful, { ...delivery, ...(reported?.url ? { reported: reported.url } : {}) }) + : succeed(step, successful); + // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. Delivery reads the + // pushed branch, so it does not keep the worktree. + const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); return { save: true, envelope: ok({ @@ -1427,25 +1471,8 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: revision: run.revision, step: { id: settledStep.id, status: settledStep.status, outcome }, ...(cleanup ? { cleanup: true } : {}), + ...(dropped.length > 0 ? { dropped } : {}), }), }; }); - if ('error' in envelope) return envelope; - const pending = await readRun(config, runId); - if (typeof pending !== 'string' && pending.status === 'running' && pending.steps.some((step) => step.status === 'delivering')) { - // Reconcile before provider I/O too: limits reached by this report must stop delivery before it opens a PR. - const checked = await leaseNextCore(config, runId, report.driver, { canTakeLease: false, - running: pending.steps.flatMap((step) => step.status === 'leased' ? [step.attempt] : []), - }); - if (!('error' in checked)) await deliverPending(config, runId); - const run = await readRun(config, runId); - const step = typeof run !== 'string' && run.steps.find((state) => 'attempt' in state && state.attempt === report.attempt); - if (step) { - envelope.data.revision = run.revision; - envelope.data.step = { id: step.id, status: step.status, outcome: step.status === 'failed' || step.status === 'cancelled' ? 'failed' : 'success' }; - if (step.status === 'succeeded' && report.changes && !branchStillUsed(run, report.changes.branch)) envelope.data.cleanup = true; - } - } - if (dropped.length) envelope.data.dropped = dropped; - return envelope; } diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 9f87e51b0..97ca4cb84 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -1096,16 +1096,15 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options } if ('lease' in next.data && room()) continue; if (inFlight.size === 0 && dying.size === 0 && !prompt && !question) { - // A wait for a call that could not take a lease says nothing about what is left; ask again with room. - if ((next.data.wait as { delivery?: boolean } | undefined)?.delivery) { - await new Promise((resolve) => setTimeout(resolve, 100)); + const wait = isRecord(next.data.wait) ? next.data.wait : {}; + // Another process of this machine delivers a step this run waits for; it settles without this driver. + if (Array.isArray(wait.delivering) && wait.delivering.length > 0) { + await new Promise((resolve) => setTimeout(resolve, pollMs)); continue; } + // A wait for a call that could not take a lease says nothing about what is left; ask again with room. if (!canTakeLease) continue; - const reason = typeof (next.data.wait as { reason?: unknown }).reason === 'string' - ? (next.data.wait as { reason: string }).reason - : 'the next step is not available'; - return { error: { code: 'busy', message: reason } }; + return { error: { code: 'busy', message: typeof wait.reason === 'string' ? wait.reason : 'the next step is not available' } }; } let timer: NodeJS.Timeout | undefined; const exited = await Promise.race([ diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts new file mode 100644 index 000000000..2dbab19a9 --- /dev/null +++ b/src/workflow/pr-delivery.ts @@ -0,0 +1,51 @@ +/** + * Pull request delivery: what it reads of the business repository and of its host, and nothing it writes to a run. The + * engine builds a job under the run lock, calls `deliverPr` once it released the lock, and applies the outcome under + * the lock again, so no network call holds the lock and this module needs nothing of the engine's. + */ +import type { PrUrl } from './pr-url.js'; +import type { WorkflowFailure } from './types.js'; + +/** Which delivery an outcome belongs to: it applies only while the step still delivers this pass of this commit. */ +export interface DeliveryKey { + step: string; + pass: number; + settledAt: number; + branch: string; + head: string; +} + +/** Everything a delivery needs, read from the run under its lock. */ +export interface DeliveryJob { + key: DeliveryKey; + runId: string; + /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ + reported?: PrUrl; +} + +/** Why a delivery opened no pull request. Each one is told with the branch that keeps the work. */ +export type DeliveryFault = { kind: 'no_provider'; branch: string }; + +export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; + +/** Delivers one branch. */ +export async function deliverPr(job: DeliveryJob): Promise { + if (job.reported) return { ok: true, pr: job.reported }; + return { ok: false, fault: { kind: 'no_provider', branch: job.key.branch } }; +} + +/** The failure a step keeps for `fault`: what happened, where the work is, and the command that tries again. */ +export function deliveryFailure(job: DeliveryJob, fault: DeliveryFault): WorkflowFailure { + const retry = `teamai workflow retry ${job.runId} ${job.key.step}`; + switch (fault.kind) { + case 'no_provider': + return { + code: 'delivery_failed', + message: `PR delivery for step "${job.key.step}" failed: TeamAI cannot open a pull request on this repository's origin. The work is kept on origin in branch ${fault.branch}. Open a pull request from it, or have the driver report changes.pr_url, then run: ${retry}`, + }; + default: { + const unknown: never = fault.kind; + return unknown; + } + } +} diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts index 02ee366ae..53e47dd22 100644 --- a/src/workflow/pr-url.ts +++ b/src/workflow/pr-url.ts @@ -1,16 +1,40 @@ -/** Public run records accept HTTPS review links on any host, but never URL credentials or session parameters. */ -export function sanitizePrUrl(value: unknown): { url: string; reason?: never } | { url?: never; reason: string } { - if (typeof value !== 'string') return { reason: 'must be a URL string' }; - if (value.length > 512) return { reason: 'exceeds 512 characters' }; +import { z } from 'zod'; + +const MAX_LENGTH = 512; + +/** + * A pull request URL safe to store and publish: https on any host and path, no credentials, no query or fragment, at + * most 512 characters. Parsing a sanitized value gives it back unchanged, so a stored URL parses to itself. + */ +export const PrUrlSchema = z.string().transform((value, ctx) => { + const clean = sanitize(value); + if ('reason' in clean) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: clean.reason }); + return z.NEVER; + } + return clean.url; +}).brand<'PrUrl'>(); +export type PrUrl = z.infer; + +function sanitize(value: string): { url: string } | { reason: string } { + if (value.length > MAX_LENGTH) return { reason: `exceeds ${MAX_LENGTH} characters` }; + let url: URL; try { - const url = new URL(value); - if (url.protocol !== 'https:') return { reason: 'must use HTTPS' }; - if (url.username || url.password) return { reason: 'contains credentials' }; - url.search = ''; - url.hash = ''; - const sanitized = url.toString(); - return sanitized.length <= 512 ? { url: sanitized } : { reason: 'exceeds 512 characters' }; + url = new URL(value); } catch { return { reason: 'is not a valid URL' }; } + if (url.protocol !== 'https:') return { reason: 'must use HTTPS' }; + if (url.username || url.password) return { reason: 'contains credentials' }; + url.search = ''; + url.hash = ''; + const sanitized = url.toString(); + return sanitized.length <= MAX_LENGTH ? { url: sanitized } : { reason: `exceeds ${MAX_LENGTH} characters` }; +} + +/** Why a reported URL was not kept is said without the value, which may hold a token. */ +export function sanitizePrUrl(value: unknown): { url: PrUrl; reason?: never } | { url?: never; reason: string } { + if (typeof value !== 'string') return { reason: 'must be a URL string' }; + const parsed = PrUrlSchema.safeParse(value); + return parsed.success ? { url: parsed.data } : { reason: parsed.error.issues[0]?.message ?? 'is not a valid URL' }; } diff --git a/src/workflow/records.ts b/src/workflow/records.ts index 8e07c517b..f82265229 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -27,7 +27,6 @@ import { installOwner, runLockPath } from './store.js'; import { settled, VISIBLE_OUTCOMES, type StepPass, type StepState, type VisibleOutcome } from './step-state.js'; import type { TokenUsage } from '../tool-launch/stdout.js'; import type { DriverIdentity, RunState, RunStatus, WorkflowFailure } from './types.js'; -import { sanitizePrUrl } from './pr-url.js'; /** * The failure codes the engine writes, then those the local driver sets. A code a model writes into its result, or @@ -38,7 +37,7 @@ const RECORDED_FAILURES = [ 'loop_exhausted', 'invalid_output', 'invalid_template', 'invalid_inputs', 'invalid_items', 'item_cycle', 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', - 'delivery_failed', 'timeout', + 'delivery_failed', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; @@ -98,7 +97,7 @@ export type WorkflowRunRecord = { stepCounts?: { total: number; succeeded: number; failed: number; skipped: number; cancelled: number }; /** Settled attempts across the run — the cumulative iteration count. */ iterations?: number; - /** Validated review links, first appearance order. Public on a public team repo; omitted before this version. */ + /** The pull requests the run delivered, in step order. Public on a public team repo; absent when it delivered none. */ prs?: string[]; username: string; publishedAt: string; @@ -171,6 +170,19 @@ function failureOf(run: RunState): RecordedFailure | undefined { return [first, ...failed.filter(under)].map((step) => step.failure).find(isRecordedFailure)?.code; } +/** + * The pull requests the run's steps delivered, once each, in the order of the steps that recorded them and, for one + * step, of its passes. That is item order for a `for_each`, whichever item finished first, and a loop that revised one + * branch lists its pull request once. A step a loop removed from the run lists first. Only what delivery recorded on a + * branch is listed, never a step's outputs. + */ +function prsOf(run: RunState): string[] { + const position = new Map(run.steps.map((step, index) => [step.id, index])); + const recorded = Object.values(run.copies ?? {}).flatMap((copy) => (copy.pr ? [copy.pr] : [])); + const ordered = recorded.sort((left, right) => (position.get(left.step) ?? -1) - (position.get(right.step) ?? -1) || left.pass - right.pass); + return [...new Set(ordered.map((pr) => pr.url))]; +} + /** * Build the end-phase record: outcome, duration, iterations, and step counts, * all derived as counts so no step payload crosses into the record. @@ -213,13 +225,7 @@ export function buildEndRecord( ]), ...(run.archive ?? []).map((archived) => entry(archived.step, archived.pass, archived)), ]; - const changes = [ - ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), - ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), - ]; - const prs = [...new Set(changes.filter((item) => item.prRecordedAt !== undefined) - .sort((a, b) => a.prRecordedAt! - b.prRecordedAt!) - .flatMap((item) => sanitizePrUrl(item.pr_url).url ?? []))]; + const prs = prsOf(run); return { ...buildStartRecord(run, driver, username, now), phase: 'end', @@ -259,6 +265,7 @@ export function mergeRecord(origin: WorkflowRunRecord | null, next: WorkflowRunR const merged = { ...origin, ...next, startedAt: origin.startedAt }; if (next.end === undefined) delete merged.end; if (next.failure === undefined) delete merged.failure; + if (next.prs === undefined) delete merged.prs; return merged; } diff --git a/src/workflow/run-controls.ts b/src/workflow/run-controls.ts index ac68f95db..aa367ad30 100644 --- a/src/workflow/run-controls.ts +++ b/src/workflow/run-controls.ts @@ -9,7 +9,7 @@ import { publishRunEnd } from './handoff.js'; import { archiveUnder, isUnder } from './loop.js'; import { publishRunRecords, removeQueuedEndRecord } from './records.js'; import { contextOf, localId, statesIn } from './scope.js'; -import { answerHitl, cancel, cancellable, deliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; +import { answerHitl, cancel, cancellable, redeliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; import type { RunState, WorkflowEnvelope } from './types.js'; async function controlRun( @@ -269,10 +269,9 @@ export async function retryStep(config: LocalConfig, runId: string, step: string const reopened = run.status !== 'running'; for (const container of containers) { if (container.child === target) { - if (target.status === 'failed' && target.failure?.code === 'delivery_failed' && target.changes) { - const { status: _status, outcome: _outcome, failure: _failure, message: _message, ...settlement } = target; - deliver(target, { ...settlement, settledAt: run.revision + 1 }); - } else retryInstance(run, target); + // A failed delivery delivers the same branch again; the agent's work it delivers does not change. + if (target.status === 'failed' && target.failure?.code === 'delivery_failed' && target.delivery) redeliver(target, target.delivery); + else retryInstance(run, target); resetIteration(target); } for (const state of container.states) { diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 885af1280..e9a104116 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -5,6 +5,7 @@ import { z } from 'zod'; import { canonicalJson } from '../utils/canonical-json.js'; import { redact } from '../utils/redact.js'; import { isRecord } from './expression.js'; +import { PrUrlSchema } from './pr-url.js'; import { ArchivedPassSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; @@ -52,8 +53,14 @@ const BodySchema = z.object({ archive: z.array(withCommitIds(ArchivedPassSchema)), round: z.number().int().optional(), base: z.object({ ref: z.string().optional(), commit: CommitIdSchema }).optional(), - /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ - copies: z.record(z.object({ commit: CommitIdSchema.optional() })), + /** + * The run's workspaces by branch, with the commit each was last reported at and the pull request it was delivered to. + * Which driver holds one stays local. + */ + copies: z.record(z.object({ + commit: CommitIdSchema.optional(), + pr: z.object({ url: PrUrlSchema, step: z.string(), pass: z.number().int() }).optional(), + })), sessionContinuations: z.array(SessionContinuationSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ scorers: z.record(z.object({ sourceHash: z.string() })).optional(), @@ -199,7 +206,7 @@ const STEP_RULES = { iteration: 'publish', round: 'publish', status: 'publish', attempt: 'publish', lease: 'rewrite', outcome: 'publish', outputs: 'publish', end: 'publish', results: 'publish', settledAt: 'publish', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', - agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', deliveryDeadline: 'publish', + agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', delivery: 'publish', } satisfies FieldRules; const PASS_RULES = { @@ -220,7 +227,7 @@ const FAILURE_RULES = { 'code': 'publish', message: 'rewrite' } satisfies FieldR // A lease another machine held is released when the run is handed over, so the session it offered stays here. const LEASE_RULES = { attempt: 'publish', revision: 'publish', driver: 'rewrite', session: 'drop' } satisfies FieldRules; -const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish' } satisfies FieldRules[string]>; +const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish', pr: 'publish' } satisfies FieldRules[string]>; const FAILURE_REWRITES = { message: () => ({ message: HANDED_FAILURE_MESSAGE }), diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 89ace6749..7c204ca3a 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -2,6 +2,7 @@ import { z } from 'zod'; import { AgentObservationSchema, UsedSchema } from '../agent-observation.js'; import { DriverSchema } from '../driver-identity.js'; import type { TokenUsage } from '../tool-launch/stdout.js'; +import { PrUrlSchema } from './pr-url.js'; import type { StepLease, WorkflowFailure } from './types.js'; import type { WorkspaceMode } from './workspace.js'; @@ -31,8 +32,6 @@ const ChangesSchema = z.object({ from: z.string(), to: z.string(), commits: z.array(z.string()), - pr_url: z.string().optional(), - prRecordedAt: z.number().int().optional(), }); /** What a read-only step inside a copy reports: the branch it ran on and the commit it read. It wrote nothing. */ @@ -129,6 +128,13 @@ const StepBaseSchema = z.object({ }); type StepBase = z.infer; +/** + * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on + * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. + */ +const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); +export type Delivery = z.infer; + /** * What a step that ran keeps. A file written before `settledAt` existed reads it as 0. `end` is the end the workflow * this step runs reached. @@ -139,8 +145,8 @@ const settledFields = { results: ResultsSchema.optional(), settledAt: z.number().int().default(0), attempt: z.string().optional(), - /** The original execution bound continues while its completed work is delivered. */ - deliveryDeadline: z.number().optional(), + /** Kept once the step settles, so a retry of a failed delivery delivers the same branch again. */ + delivery: DeliverySchema.optional(), ...report, }; @@ -153,13 +159,14 @@ const settledFields = { const PendingSchema = StepBaseSchema.extend({ status: z.literal('pending') }); const WaitingSchema = StepBaseSchema.extend({ status: z.literal('waiting') }); const RunningSchema = StepBaseSchema.extend({ status: z.literal('running') }); -const DeliveringSchema = StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields }); +/** `delivering`: the agent succeeded and its work is on its branch; dependents wait until that branch has its pull request. */ +const DeliveringSchema = StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema }); const SucceededSchema = StepBaseSchema.extend({ status: z.literal('succeeded'), outcome: z.literal('success').default('success'), ...settledFields, loopFinal: z.boolean().optional(), }); const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional() }); const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); -const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }); +const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema }); const LeasedSchema = StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }); /** @@ -187,6 +194,7 @@ type StepIn = Extract; type PendingStep = StepIn<'pending'>; export type LeasedStep = StepIn<'leased'>; type RunningStep = StepIn<'running'>; +export type DeliveringStep = StepIn<'delivering'>; type SucceededStep = StepIn<'succeeded'>; type FailedStep = StepIn<'failed'>; type SkippedStep = StepIn<'skipped'>; @@ -280,22 +288,37 @@ export function succeed(state: StepState, settlement: Settlement) return become(state, { status: 'succeeded', outcome: 'success', ...settlement }); } -/** Agent work is durable, but dependents wait for delivery to finish. */ -export function deliver(state: StepState, settlement: Settlement, deadline?: number): StepIn<'delivering'> { - return become>(state, { status: 'delivering', ...settlement, ...(deadline !== undefined ? { deliveryDeadline: deadline } : {}) }); +/** The agent's work is settled and durable; the step succeeds once `delivery` has its pull request. */ +export function startDelivery(state: StepState, settlement: Settlement, delivery: Delivery): DeliveringStep { + return become(state, { status: 'delivering', ...settlement, delivery }); +} + +/** A delivered step settles as its agent left it, at the revision the agent's report settled it. */ +export function delivered(state: DeliveringStep): SucceededStep { + const { id: _id, status: _status, ...settlement } = state; + return succeed(state, settlement); +} + +/** A delivery that failed stops the step, keeping what its agent did and the branch a retry delivers again. */ +export function deliveryFailed(state: DeliveringStep, failure: WorkflowFailure): FailedStep { + const { id: _id, status: _status, ...settlement } = state; + return fail(state, { ...settlement, message: failure.message, failure }); +} + +/** A failed delivery is retried in the same pass and attempt: the agent does not run again. */ +export function redeliver(state: FailedStep, delivery: Delivery): DeliveringStep { + const { id: _id, status: _status, outcome: _outcome, failure: _failure, message: _message, ...settlement } = state; + return startDelivery(state, settlement, delivery); } export function fail(state: StepState, settlement: Settlement): FailedStep { return become(state, { status: 'failed', outcome: 'failed', ...settlement }); } -/** An open step is cancelled before it settles. A leased one keeps its invalidated attempt. */ +/** An open step is cancelled before it settles. A leased or delivering one keeps the attempt it no longer settles. */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { - // Delivery follows completed agent work. Cancellation must retain that work's usage and any review link. - const reported = state.status === 'delivering' - ? Object.fromEntries(SETTLED_KEYS.filter((key) => state[key] !== undefined).map((key) => [key, state[key]])) - : state.status === 'leased' ? { attempt: state.attempt } : {}; - return become(state, { ...reported, status: 'cancelled', settledAt, failure }); + const attempt = state.status === 'leased' || state.status === 'delivering' ? state.attempt : undefined; + return become(state, { status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), failure }); } /** The current pass of a settled step, as `passes` keeps it once a loop runs the step again; undefined while it has not settled. */ diff --git a/src/workflow/store.ts b/src/workflow/store.ts index 46b62c96a..0e8a81b92 100644 --- a/src/workflow/store.ts +++ b/src/workflow/store.ts @@ -41,6 +41,11 @@ export function runLockPath(config: LocalConfig, runId: string): string { return path.join(runsDir(config), `${runId}.lock`); } +/** Held while this machine delivers a run's pull requests: never the run lock, which no network call may hold. */ +export function deliveryLockPath(config: LocalConfig, runId: string): string { + return path.join(runsDir(config), `${runId}.delivery.lock`); +} + export function driverLockPath(config: LocalConfig, runId: string): string { return path.join(runsDir(config), runId, 'driver.lock'); } diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 08d0f8684..77f231273 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,9 +1,17 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; +import type { PrUrl } from './pr-url.js'; import type { ArchivedPass, SessionContinuation, StepState } from './step-state.js'; import type { LeaseSession } from './workspace.js'; +/** The pull request a branch was delivered to, and the step pass whose delivery recorded it. */ +export interface BranchPr { + url: PrUrl; + step: string; + pass: number; +} + /** The driver's lease ownership, independent of its per-request capabilities. */ export interface DriverIdentity { kind: string; @@ -256,9 +264,10 @@ export interface RunState { checkout?: string; /** * Workspaces this run has handed out, by branch. A `continue` step stays with the driver that opened the copy. A copy a - * claim brought from another machine has no owner until a driver leases it. + * claim brought from another machine has no owner until a driver leases it. `pr` is the pull request delivered for the + * branch, written once by the step whose delivery first recorded it: one branch has one pull request. */ - copies?: Record; + copies?: Record; /** * Sessions a step resumed and continued under another id, in report order: a copy on another machine or in another * tool. `to` holds the turns of `from` and the ones that step added, so a step that would resume `from` resumes `to`. diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index a02c33aa1..75ee0ea81 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -5,7 +5,7 @@ import { itemsAfter, itemsOf } from './fan-out.js'; import { localId, statesIn, stepDefinitions, type Scope } from './scope.js'; import { parseStepPath } from './step-path.js'; -import { passOf, type StepState } from './step-state.js'; +import { passOf, type Delivery, type StepState } from './step-state.js'; import type { DriverIdentity, RunState, WorkflowDefinition, WorkflowFailure, WorkflowStep } from './types.js'; export type WorkspaceMode = 'read-only' | 'new' | 'continue'; @@ -17,9 +17,6 @@ export interface StepChanges { from: string; to: string; commits: string[]; - pr_url?: string; - /** Engine revision at which the validated URL first appeared. Never accepted from a driver. */ - prRecordedAt?: number; } /** @@ -153,6 +150,24 @@ function changesOf(state: StepState | undefined): StepChanges | undefined { return state.changes; } +/** + * What a step that just succeeded delivers, or undefined when its branch holds nothing beyond the run base. A leaf + * delivers the branch it reported. A workflow step delivers its own copy once, never per step inside it; one without a + * copy of its own, such as a `for_each` item that runs a workflow, delivers the branch its latest step wrote. + */ +export function deliverableOf(run: RunState, state: StepState, step: WorkflowStep, changes?: StepChanges): Omit | undefined { + let branch = changes?.branch; + let head = changes?.to; + if (step.workflow !== undefined) { + const own = branchName(run.id, state.id, state.pass ?? 1); + const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && changesOf(item)) + .sort((left, right) => ('settledAt' in right ? right.settledAt : 0) - ('settledAt' in left ? left.settledAt : 0))[0]; + branch = run.copies?.[own]?.commit !== undefined ? own : changesOf(latest)?.branch; + head = branch === undefined ? undefined : run.copies?.[branch]?.commit; + } + return branch === undefined || head === undefined || head === run.base?.commit ? undefined : { branch, head }; +} + /** Direct steps of a workflow step, such as `work.write`, not its fan-out items. */ function workflowChildren(run: RunState, step: StepState): StepState[] { const prefix = `${step.id}.`; From 7cad04688352387f82c29e672c64362f67d76681 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 11:30:15 +0200 Subject: [PATCH 07/77] feat(workflow): name a dropped pull request URL without echoing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A driver that reports changes.pr_url learns when it was not kept: the report is still accepted, and the envelope lists the field under dropped with a reason and never the value, since it may hold a token. A URL from a step without deliver: pr is dropped the same way rather than discarded in silence, as nothing would ever read it. The sanitizer table now asserts the reason for each dropped row, and the fast-check properties generate credentials, queries and fragments, which the previous generator never produced, so the properties that none of them survive could not fail. The credentials case runs on a delivering step and checks the end record as well as the envelope and the run file. The conformance report operation takes the extra changes fields directly, so its harnesses parse the saved lease instead of asserting its workspace exists. Red: "drops a URL from a step that does not deliver, saying why, and still accepts the report" failed with "expected { data: { accepted: true, …(3) } } to deeply equal { data: { accepted: true, …(4) } }" (no dropped entry). The strengthened properties failed with counterexample "https://a.aa?a" against a sanitizer that kept the query. --- .../e2e/workflow-conformance.test.ts | 18 +++++- src/__tests__/workflow-conformance.test.ts | 18 +++++- src/__tests__/workflow-delivery.test.ts | 62 ++++++++++++++----- src/workflow/conformance.ts | 8 +-- src/workflow/engine.ts | 6 +- 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/__tests__/e2e/workflow-conformance.test.ts b/src/__tests__/e2e/workflow-conformance.test.ts index 6ce9724e7..3951b5ef4 100644 --- a/src/__tests__/e2e/workflow-conformance.test.ts +++ b/src/__tests__/e2e/workflow-conformance.test.ts @@ -4,6 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; import { resolveProjectDataHome } from '../../config.js'; import { workflowConformanceScenarios, workflowGitConformanceScenarios, type GitConformanceName, type WorkflowClient } from '../../workflow/conformance.js'; import type { StepState } from '../../workflow/step-state.js'; @@ -14,6 +15,17 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '. const CLI = path.join(ROOT, 'dist', 'index.js'); const driver: DriverIdentity = { kind: 'local', machine: 'cli-conformance', workspace: '/work' }; +const SavedLeaseSchema = z.object({ + attempt: z.string(), revision: z.number(), workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), +}); +type SavedLease = z.infer; + +/** The `changes` a report of `lease` carries: the branch and base of its workspace, and the scenario's own fields. */ +function leasedChanges(lease: SavedLease | undefined, fields: Record): Record { + if (!lease?.workspace) throw new Error('A report with changes needs a saved lease that ran in a workspace'); + return { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...fields }; +} + function runCLI(args: string[], cwd: string, env: NodeJS.ProcessEnv = process.env): Promise<{ code: number | null; stdout: string; stderr: string }> { return new Promise((resolve) => { const child = spawn(process.execPath, [CLI, ...args], { @@ -75,7 +87,7 @@ describe('workflow protocol conformance through the CLI', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -99,7 +111,7 @@ describe('workflow protocol conformance through the CLI', () => { if (operation.cancelled) expect('data' in value && value.data.cancelled).toEqual(operation.cancelled.map(attemptOf)); for (const field of operation.absent ?? []) expect('data' in value ? value.data.lease : undefined).not.toHaveProperty(field); if ('data' in value && value.data.lease) { - leases.set(operation.saveAs, value.data.lease as { attempt: string; revision: number }); + leases.set(operation.saveAs, SavedLeaseSchema.parse(value.data.lease)); } continue; } @@ -115,7 +127,7 @@ describe('workflow protocol conformance through the CLI', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, - ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), + ...(operation.changes ? { changes: leasedChanges(lease, operation.changes) } : {}), }); expect(value).toMatchObject(operation.expect); } diff --git a/src/__tests__/workflow-conformance.test.ts b/src/__tests__/workflow-conformance.test.ts index f790ffa34..7cf6bffe7 100644 --- a/src/__tests__/workflow-conformance.test.ts +++ b/src/__tests__/workflow-conformance.test.ts @@ -2,6 +2,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; +import { z } from 'zod'; import type { LocalConfig } from '../types.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; import { workflowConformanceScenarios, type WorkflowClient } from '../workflow/conformance.js'; @@ -11,6 +12,17 @@ import { readRun } from '../workflow/store.js'; import type { DriverIdentity } from '../workflow/types.js'; const dirs: string[] = []; + +const SavedLeaseSchema = z.object({ + attempt: z.string(), revision: z.number(), workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), +}); +type SavedLease = z.infer; + +/** The `changes` a report of `lease` carries: the branch and base of its workspace, and the scenario's own fields. */ +function leasedChanges(lease: SavedLease | undefined, fields: Record): Record { + if (!lease?.workspace) throw new Error('A report with changes needs a saved lease that ran in a workspace'); + return { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...fields }; +} const driver: DriverIdentity = { kind: 'local', machine: 'conformance', workspace: '/work' }; afterEach(() => { @@ -64,7 +76,7 @@ describe('workflow protocol conformance in process', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -86,7 +98,7 @@ describe('workflow protocol conformance in process', () => { if (operation.cancelled) expect('data' in envelope && envelope.data.cancelled).toEqual(operation.cancelled.map(attemptOf)); for (const field of operation.absent ?? []) expect('data' in envelope ? envelope.data.lease : undefined).not.toHaveProperty(field); if ('data' in envelope && envelope.data.lease) { - leases.set(operation.saveAs, envelope.data.lease as { attempt: string; revision: number }); + leases.set(operation.saveAs, SavedLeaseSchema.parse(envelope.data.lease)); } continue; } @@ -102,7 +114,7 @@ describe('workflow protocol conformance in process', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, - ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), + ...(operation.changes ? { changes: leasedChanges(lease, operation.changes) } : {}), }); expect(envelope).toMatchObject(operation.expect); } diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 6f514f997..80a5b0e21 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -115,24 +115,37 @@ describe('workflow PR delivery contract', () => { }); it.each([ - ['https://github.com/acme/app/pull/101', 'https://github.com/acme/app/pull/101'], - ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview'], - ['https://review.acme.internal/changes/4821?session=abc#c3', 'https://review.acme.internal/changes/4821'], - ['https://user:token@github.com/acme/app/pull/1', undefined], - ['http://github.com/acme/app/pull/1', undefined], ['not a url', undefined], - [`https://example.com/${'a'.repeat(512)}`, undefined], - ])('sanitizes %s', (value, expected) => expect(sanitizePrUrl(value).url).toBe(expected)); - - it('never preserves credentials, queries or fragments and is idempotent and bounded', () => { - fc.assert(fc.property(fc.webUrl({ validSchemes: ['https', 'http'] }), (value) => { + ['https://github.com/acme/app/pull/101', { url: 'https://github.com/acme/app/pull/101' }], + ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', { url: 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview' }], + ['https://review.acme.internal/changes/4821?session=abc#c3', { url: 'https://review.acme.internal/changes/4821' }], + ['https://user:token@github.com/acme/app/pull/1', { reason: 'contains credentials' }], + ['http://github.com/acme/app/pull/1', { reason: 'must use HTTPS' }], + ['not a url', { reason: 'is not a valid URL' }], + [`https://example.com/${'a'.repeat(512)}`, { reason: 'exceeds 512 characters' }], + [101, { reason: 'must be a URL string' }], + ])('sanitizes %s', (value, expected) => expect(sanitizePrUrl(value)).toEqual(expected)); + + const urls = fc.webUrl({ validSchemes: ['https', 'http'], withQueryParameters: true, withFragments: true, authoritySettings: { withUserInfo: true } }); + + it('never keeps credentials, a query or a fragment, gives back what it kept unchanged, and stays within 512 characters', () => { + fc.assert(fc.property(fc.oneof(urls, fc.string(), fc.string({ minLength: 513, maxLength: 700 })), (value) => { const result = sanitizePrUrl(value); - if (!result.url) return; + if (result.url === undefined) return; const url = new URL(result.url); expect(url.protocol).toBe('https:'); expect([url.username, url.password, url.search, url.hash]).toEqual(['', '', '', '']); expect(result.url.length).toBeLessThanOrEqual(512); expect(sanitizePrUrl(result.url)).toEqual(result); - }), { numRuns: 300 }); + }), { numRuns: 500 }); + }); + + it('drops every URL with credentials, and keeps the host and path of any other https URL', () => { + fc.assert(fc.property(urls, (value) => { + const url = new URL(value); + const expected = url.username || url.password ? undefined + : url.protocol === 'https:' && `${url.origin}${url.pathname}`.length <= 512 ? `${url.origin}${url.pathname}` : undefined; + expect(sanitizePrUrl(value).url).toBe(expected); + }), { numRuns: 500 }); }); }); @@ -209,12 +222,27 @@ describe('delivering a step', () => { }); describe('reported URLs from a driver', () => { - it('drops a URL with credentials without failing the report, and never stores or echoes it', async () => { + it('drops a URL with credentials without failing the report, and never stores, echoes or publishes it', async () => { const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest'); - const { config, runId } = await fixture(` - id: write\n command: node write.js\n workspace: new\n`); - const accepted = await succeed(config, runId, await take(config, runId), { pr_url: 'https://user:token@github.com/fixture/app/pull/1' }); - expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); - expect(JSON.stringify([accepted, await state(config, runId)])).not.toContain('token'); + const { config, runId } = await fixture(FIX_THEN_REVIEW); + const accepted = await succeed(config, runId, await take(config, runId), { pr_url: 'https://user:s3cret-value@github.com/fixture/app/pull/1', extra: 'kept-out' }); + expect(accepted).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, + dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }], ignored: ['changes.extra'], + } }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).not.toHaveProperty('delivery.reported'); + expect(JSON.stringify([accepted, run, buildEndRecord(run, remote, 'tester')])).not.toMatch(/s3cret-value|kept-out/); expect(validate).not.toHaveBeenCalled(); }); + + it('drops a URL from a step that does not deliver, saying why, and still accepts the report', async () => { + const { config, runId } = await fixture(' - id: write\n command: node write.js\n workspace: new\n'); + expect(await succeed(config, runId, await take(config, runId), { pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ data: { + accepted: true, revision: 2, step: { id: 'write', status: 'succeeded', outcome: 'success' }, cleanup: true, + dropped: [{ field: 'changes.pr_url', reason: 'the step has no deliver: pr' }], + } }); + expect((await state(config, runId)).copies?.[`teamai/${runId}/write/p1`]?.pr).toBeUndefined(); + }); }); diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index 5424e30a7..a80a9a7f3 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -37,8 +37,8 @@ export type ConformanceOperation = lease?: string; attempt?: string; revision?: number; - /** The harness fills the changes branch/base from this operation's saved lease. */ - leasedChanges?: boolean; + /** Fields added to the `changes` the harness reports from the workspace of this operation's saved lease. */ + changes?: Record; body: Record; expect: Record; }; @@ -266,7 +266,7 @@ export const workflowConformanceScenarios: ConformanceScenario[] = [ workflowName: 'reported-pr', yaml: REPORTED_PR, operations: [ { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, - { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' } }, expect: { data: { accepted: true, step: { id: 'work', status: 'delivering', outcome: 'success' }, ignored: ['changes.unknown'] } } }, + { kind: 'report', lease: 'work', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' }, body: { outcome: 'success' }, expect: { data: { accepted: true, step: { id: 'work', status: 'delivering', outcome: 'success' }, ignored: ['changes.unknown'] } } }, { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, ], expectState: { steps: [{ id: 'work', status: 'succeeded', delivery: { reported: 'https://review.example/changes/1' } }] }, @@ -275,7 +275,7 @@ export const workflowConformanceScenarios: ConformanceScenario[] = [ name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, operations: [ { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, - { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://user:token@review.example/changes/1' } }, expect: { data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } } }, + { kind: 'report', lease: 'work', changes: { pr_url: 'https://user:token@review.example/changes/1' }, body: { outcome: 'success' }, expect: { data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } } }, { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, ], }, diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 5ed9ab022..2d449b11e 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1379,8 +1379,6 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: } const install = await installOwner(config); const reported = report.prUrl === undefined ? undefined : sanitizePrUrl(report.prUrl); - // A URL that is not kept is named without its value, which may hold a token, and the report still counts. - const dropped = reported?.reason === undefined ? [] : [{ field: 'changes.pr_url', reason: reported.reason }]; return withRun(config, runId, async (run) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { @@ -1464,6 +1462,8 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. Delivery reads the // pushed branch, so it does not keep the worktree. const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); + // A URL that is not kept is named without its value, which may hold a token, and the report still counts. + const dropReason = reported?.reason ?? (reported && defined?.deliver !== 'pr' ? 'the step has no deliver: pr' : undefined); return { save: true, envelope: ok({ @@ -1471,7 +1471,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: revision: run.revision, step: { id: settledStep.id, status: settledStep.status, outcome }, ...(cleanup ? { cleanup: true } : {}), - ...(dropped.length > 0 ? { dropped } : {}), + ...(dropReason ? { dropped: [{ field: 'changes.pr_url', reason: dropReason }] } : {}), }), }; }); From 455b6383fc4a809d1cfc6e91ae7450ccb16a1dd3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 11:42:31 +0200 Subject: [PATCH 08/77] feat(workflow): refuse to deliver a branch that changes protected paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI and hook configuration run with someone's credentials, and agent configuration steers the next agent, so a pull request TeamAI delivers may not change either. Delivery now compares the delivered head with the run base in the run's checkout, fetching the run's branches when the head is not there yet, and fails with protected_path naming the files before any provider request. A URL a driver reported is checked the same way, so no step succeeds on such a branch. Changes a branch inherited from a step it needs count as well, since the diff is against the base. The protected list has one owner in worktree.ts: AGENT_PATHS, which steps already read from the run base, plus DELIVERY_ONLY_PATHS for CI and hooks, plus AGENTS.md and CLAUDE.md at any depth. The head comes from a driver's report, so it is read only as a full commit id; a ref name such as main would otherwise diff another tree and pass. The reported-URL conformance scenario now stops at what a driver sees, the report and the URL kept on the delivering step, because its runs have no repository to read the branch from; real-git tests cover the delivery itself. Red: "refuses a branch that changes protected paths, naming them, before any provider request" failed for both cases, with "expected [ { id: 'fix', …(9) }, …(1) ] to match object" (delivery_failed rather than protected_path) and, for a reported URL, "expected { data: { lease: … } }" (the step succeeded and review was leased); "protect every path …" failed with "DELIVERY_ONLY_PATHS is not iterable". The full commit id test failed with "expected { data: { lease: … } }" with the id check removed. --- src/__tests__/workflow-delivery.test.ts | 98 +++++++++++++++++++++++-- src/workflow/conformance.ts | 5 +- src/workflow/engine.ts | 6 +- src/workflow/pr-delivery.ts | 67 +++++++++++++++-- src/workflow/records.ts | 2 +- src/workflow/worktree.ts | 18 +++++ 6 files changed, 175 insertions(+), 21 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 80a5b0e21..24cac2928 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -12,6 +12,7 @@ import { syncWorkflowDefinitions } from '../workflow/cache.js'; import { deliveryLockPath, readRun } from '../workflow/store.js'; import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; import { acquireLock, releaseLock } from '../update.js'; +import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; import { GitHubProvider } from '../providers/github/index.js'; import { buildEndRecord } from '../workflow/records.js'; import { cancelRun } from '../workflow/run-controls.js'; @@ -85,11 +86,37 @@ async function report(config: LocalConfig, runId: string, body: Record = { 'work.txt': 'work\n' }): string { + if (!lease.workspace) throw new Error(`lease ${lease.step} has no workspace`); + const clone = fs.mkdtempSync(path.join(root, 'driver-')); + git(root, ['clone', '-q', path.join(root, 'remote.git'), clone]); + git(clone, ['checkout', '-q', '-b', lease.workspace.branch, lease.workspace.base.commit]); + for (const [file, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(clone, file)), { recursive: true }); + fs.writeFileSync(path.join(clone, file), content); + } + git(clone, ['add', '-A']); + git(clone, ['-c', 'user.name=Driver', '-c', 'user.email=driver@example.com', 'commit', '-qm', 'work']); + git(clone, ['push', '-q', 'origin', lease.workspace.branch]); + return git(clone, ['rev-parse', 'HEAD']); +} + +/** Reports a lease's success with one commit on its branch (`to`, a commit no repository has unless given), and `changes` fields the driver adds. */ async function succeed(config: LocalConfig, runId: string, lease: Lease, changes: Record = {}) { const base = lease.workspace?.base.commit; + const to = typeof changes.to === 'string' ? changes.to : head; return report(config, runId, { attempt: lease.attempt, revision: lease.revision, outcome: 'success', - ...(lease.workspace ? { changes: { branch: lease.workspace.branch, base, from: base, to: head, commits: [head], ...changes } } : {}) }); + ...(lease.workspace ? { changes: { branch: lease.workspace.branch, base, from: base, to, commits: [to], ...changes } } : {}) }); +} + +/** The pull request lookups and creates a GitHub origin could see, none of which may run. */ +function provider() { + return [ + vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockRejectedValue(new Error('no PR may be opened')), + vi.spyOn(GitHubProvider.prototype, 'findOpenPullRequest').mockRejectedValue(new Error('no PR may be looked up')), + vi.spyOn(GitHubProvider.prototype, 'validatePullRequest').mockRejectedValue(new Error('no PR may be read')), + ]; } const FIX_THEN_REVIEW = ` - id: fix\n workspace: new\n command: node fix.js\n deliver: pr\n - id: review\n needs: [fix]\n command: echo review\n`; @@ -152,12 +179,12 @@ describe('workflow PR delivery contract', () => { describe('delivering a step', () => { it('delivers a driver-reported URL without opening a PR, and leases the dependent in the same next', async () => { const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockRejectedValue(new Error('no PR may be opened')); - const { config, runId } = await fixture(FIX_THEN_REVIEW); + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); const branch = `teamai/${runId}/fix/p1`; expect(fix).toMatchObject({ step: 'fix', workspace: { branch } }); - expect(await succeed(config, runId, fix, { pr_url: 'https://review.acme.internal/changes/4821?session=abc#c3' })) + expect(await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://review.acme.internal/changes/4821?session=abc#c3' })) .toEqual({ data: { accepted: true, revision: 2, step: { id: 'fix', status: 'delivering', outcome: 'success' }, cleanup: true } }); const held = deliveryLockPath(config, runId); @@ -181,8 +208,9 @@ describe('delivering a step', () => { }); it('keeps the local driver polling while another process delivers, rather than giving up as busy', async () => { - const { config, cwd, runId } = await fixture(FIX_THEN_REVIEW); - await succeed(config, runId, await take(config, runId), { pr_url: 'https://review.acme.internal/changes/1' }); + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://review.acme.internal/changes/1' }); const held = deliveryLockPath(config, runId); expect(await acquireLock(held)).toBe(true); const released = new Promise((resolve) => setTimeout(() => void releaseLock(held).then(resolve), 200)); @@ -194,12 +222,14 @@ describe('delivering a step', () => { it('fails a delivery that has no URL and no host to open one on, naming the branch that keeps the work', async () => { const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); git(cwd, ['remote', 'set-url', 'origin', path.join(root, 'remote.git')]); - expect(await succeed(config, runId, await take(config, runId))).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); + const fix = await take(config, runId); + const to = pushWork(root, fix); + expect(await succeed(config, runId, fix, { to })).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const branch = `teamai/${runId}/fix/p1`; expect((await state(config, runId)).steps).toMatchObject([ - { id: 'fix', status: 'failed', delivery: { branch, head }, failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }, + { id: 'fix', status: 'failed', delivery: { branch, head: to }, failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }, { id: 'review', status: 'skipped' }, ]); expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); @@ -216,11 +246,63 @@ describe('delivering a step', () => { expect(run.copies?.[`teamai/${runId}/fix/p1`]?.pr).toBeUndefined(); }); + it.each([ + ['TeamAI would open the pull request', {}], + ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], + ])('refuses a branch that changes protected paths, naming them, before any provider request, when %s', async (_case, changes) => { + const calls = provider(); + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const to = pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n', 'src/app.ts': 'ok\n', 'packages/web/CLAUDE.md': 'rules\n' }); + await succeed(config, runId, fix, { to, ...changes }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('.github/workflows/ci.yml, packages/web/CLAUDE.md.') } }, + { id: 'review', status: 'skipped' }, + ]); + expect(run.steps[0]).not.toMatchObject({ failure: { message: expect.stringContaining('src/app.ts') } }); + expect(run.copies?.[`teamai/${runId}/fix/p1`]?.pr).toBeUndefined(); + for (const call of calls) expect(call).not.toHaveBeenCalled(); + }); + + it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n' }); + await succeed(config, runId, fix, { to: 'main', pr_url: 'https://github.com/fixture/app/pull/1' }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ + status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`Push branch teamai/${runId}/fix/p1 to origin`) }, + }); + }); + + it('refuses protected changes the branch inherited from a step it needs, though the delivering step left them alone', async () => { + const calls = provider(); + const write = (file: string) => `node -e "require('fs').mkdirSync(require('path').dirname('${file}'),{recursive:true});require('fs').writeFileSync('${file}','x')"`; + const { config, cwd, runId } = await fixture(` - id: earlier\n workspace: new\n command: ${write('.github/workflows/ci.yml')}\n - id: later\n needs: [earlier]\n workspace: new\n command: ${write('change.txt')}\n deliver: pr\n`); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ id: 'later', status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('.github/workflows/ci.yml') } }); + for (const call of calls) expect(call).not.toHaveBeenCalled(); + }); + it.todo('delivers a workflow step with workspace: new once, against the delivered branch of the step it needs'); it.todo('delivers each for_each item against the delivered branch of the item it waited for, and lists the PRs in item order'); it.todo('targets the nearest delivered branch, passing over a need that was skipped'); }); +describe('protected paths', () => { + it('protect every path a step reads from the run base, every CI and hook path, and AGENTS.md or CLAUDE.md at any depth, in any case', () => { + const changed = [...AGENT_PATHS, ...DELIVERY_ONLY_PATHS].flatMap((root) => [root, `${root}/nested/file`, root.toUpperCase()]); + expect(protectedPaths(changed)).toEqual(changed); + expect(protectedPaths(['src/app.ts', 'docs/agents.mdx', '.claudeignore', '.github/dependabot.yml', '.github/workflows-notes.md', 'packages/web/AGENTS.md', 'Claude.md'])) + .toEqual(['packages/web/AGENTS.md', 'Claude.md']); + }); +}); + describe('reported URLs from a driver', () => { it('drops a URL with credentials without failing the report, and never stores, echoes or publishes it', async () => { const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest'); diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index a80a9a7f3..34c8d91c9 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -262,14 +262,13 @@ steps: export const workflowConformanceScenarios: ConformanceScenario[] = [ { - name: 'a driver reports the pull request it opened, which delivers the step without opening another', + name: 'a pull request a driver reports is kept, without its query or fragment, for the step to deliver', workflowName: 'reported-pr', yaml: REPORTED_PR, operations: [ { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, { kind: 'report', lease: 'work', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' }, body: { outcome: 'success' }, expect: { data: { accepted: true, step: { id: 'work', status: 'delivering', outcome: 'success' }, ignored: ['changes.unknown'] } } }, - { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, ], - expectState: { steps: [{ id: 'work', status: 'succeeded', delivery: { reported: 'https://review.example/changes/1' } }] }, + expectState: { steps: [{ id: 'work', status: 'delivering', delivery: { reported: 'https://review.example/changes/1' } }] }, }, { name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 2d449b11e..dae1aaa1f 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -900,7 +900,11 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; const step = run.steps.find((item): item is DeliveringStep => item.status === 'delivering'); if (!step) return undefined; - return { key: deliveryKey(step), runId: run.id, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}) }; + return { + key: deliveryKey(step), runId: run.id, + ...(run.checkout ? { checkout: run.checkout } : {}), ...(run.base ? { base: run.base.commit } : {}), + ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), + }; } /** diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 2dbab19a9..be6673f69 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -5,6 +5,7 @@ */ import type { PrUrl } from './pr-url.js'; import type { WorkflowFailure } from './types.js'; +import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; /** Which delivery an outcome belongs to: it applies only while the step still delivers this pass of this commit. */ export interface DeliveryKey { @@ -19,32 +20,82 @@ export interface DeliveryKey { export interface DeliveryJob { key: DeliveryKey; runId: string; + /** The checkout the run started or was claimed in, which reads the delivered branch. */ + checkout?: string; + /** The run base commit the delivered branch is compared with. */ + base?: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ reported?: PrUrl; } -/** Why a delivery opened no pull request. Each one is told with the branch that keeps the work. */ -export type DeliveryFault = { kind: 'no_provider'; branch: string }; +/** Why a delivery opened no pull request. */ +export type DeliveryFault = + | { kind: 'no_checkout' } + | { kind: 'unreadable' } + | { kind: 'protected_path'; files: string[] } + | { kind: 'no_provider' }; export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; -/** Delivers one branch. */ +/** A full object id: the head comes from a driver's report, and anything else could reach git as an option. */ +const OBJECT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/i; + +/** + * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, + * so no pull request goes out, and no step succeeds, with a protected path changed. + */ export async function deliverPr(job: DeliveryJob): Promise { + const changed = await changedFiles(job); + if ('fault' in changed) return { ok: false, fault: changed.fault }; + const files = protectedPaths(changed.files); + if (files.length > 0) return { ok: false, fault: { kind: 'protected_path', files } }; if (job.reported) return { ok: true, pr: job.reported }; - return { ok: false, fault: { kind: 'no_provider', branch: job.key.branch } }; + return { ok: false, fault: { kind: 'no_provider' } }; +} + +/** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ +async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promise<{ files: string[] } | { fault: DeliveryFault }> { + if (!checkout || !base) return { fault: { kind: 'no_checkout' } }; + if (!OBJECT_ID.test(key.head) || !OBJECT_ID.test(base)) return { fault: { kind: 'unreadable' } }; + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [base, key.head] }); + if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; + const diff = await git(checkout, ['diff', '--no-renames', '--name-only', '-z', base, key.head, '--']); + if (diff.code !== 0) return { fault: { kind: 'unreadable' } }; + return { files: diff.stdout.split('\0').filter(Boolean) }; } -/** The failure a step keeps for `fault`: what happened, where the work is, and the command that tries again. */ +const LISTED_FILES = 20; + +/** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ export function deliveryFailure(job: DeliveryJob, fault: DeliveryFault): WorkflowFailure { - const retry = `teamai workflow retry ${job.runId} ${job.key.step}`; + const { step, branch } = job.key; + const retry = `teamai workflow retry ${job.runId} ${step}`; switch (fault.kind) { + case 'no_checkout': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the run records no checkout and base to compare branch ${branch} with, so its changes cannot be checked. The work is kept in branch ${branch}. Open a pull request from it yourself, or start the run again from a checkout of its repository.`, + }; + case 'unreadable': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the run's checkout cannot read the commit branch ${branch} reported, or the run base, even after fetching the run's branches from origin, so its changes cannot be checked. Push branch ${branch} to origin, then run: ${retry}`, + }; + case 'protected_path': { + const listed = fault.files.slice(0, LISTED_FILES).join(', '); + const more = fault.files.length > LISTED_FILES ? `, and ${fault.files.length - LISTED_FILES} more` : ''; + return { + code: 'protected_path', + message: `Step "${step}" changed protected paths against the run base: ${listed}${more}. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. Have the step leave those paths alone, then run: ${retry}`, + }; + } case 'no_provider': return { code: 'delivery_failed', - message: `PR delivery for step "${job.key.step}" failed: TeamAI cannot open a pull request on this repository's origin. The work is kept on origin in branch ${fault.branch}. Open a pull request from it, or have the driver report changes.pr_url, then run: ${retry}`, + message: `PR delivery for step "${step}" failed: TeamAI cannot open a pull request on this repository's origin. The work is kept on origin in branch ${branch}. Open a pull request from it, or have the driver report changes.pr_url, then run: ${retry}`, }; default: { - const unknown: never = fault.kind; + const unknown: never = fault; return unknown; } } diff --git a/src/workflow/records.ts b/src/workflow/records.ts index f82265229..5e7b334ca 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -37,7 +37,7 @@ const RECORDED_FAILURES = [ 'loop_exhausted', 'invalid_output', 'invalid_template', 'invalid_inputs', 'invalid_items', 'item_cycle', 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', - 'delivery_failed', + 'delivery_failed', 'protected_path', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 1a945b372..089cc494d 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -35,6 +35,24 @@ function configRoot(relative: string): string { return first !== relative && first.startsWith('.') && first !== '.github' ? first : relative; } +/** Paths a delivered branch may not change beyond AGENT_PATHS: CI and hooks, which run with someone's credentials. */ +export const DELIVERY_ONLY_PATHS: readonly string[] = [ + '.github/workflows', '.github/actions', '.github/CODEOWNERS', '.gitlab-ci.yml', '.gitlab', '.coding-ci.yaml', '.cnb.yml', '.cnb', + '.circleci', '.buildkite', '.husky', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml', '.pre-commit-config.yaml', +]; + +/** + * The files among `changed` that no pull request TeamAI delivers may change: AGENT_PATHS, DELIVERY_ONLY_PATHS, and + * AGENTS.md or CLAUDE.md at any depth. Case-insensitive, as a checkout on a case-insensitive file system reads them. + */ +export function protectedPaths(changed: readonly string[]): string[] { + const roots = [...AGENT_PATHS, ...DELIVERY_ONLY_PATHS].map((root) => root.toLowerCase()); + return changed.filter((file) => { + const lower = file.toLowerCase(); + return /(^|\/)(agents|claude)\.md$/.test(lower) || roots.some((root) => lower === root || lower.startsWith(`${root}/`)); + }); +} + export interface GitResult { code: number; stdout: string; From 2e4240e953403fa50a4b0665bdcf5bbdefa9a443 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:02:16 +0200 Subject: [PATCH 09/77] fix(workflow): keep delivered pull requests as run receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pull request recorded once per branch could not record a second request opened after a person closed the first, and it read as a cache of what was still open. The run now keeps an append-only list of receipts, deliveries, one per URL in the order each first appeared, with the branch, base branch, step and pass that delivered it. Whether a delivery reuses an open request stays the host's lookup to answer. The end record lists the receipt URLs in that order, and the snapshot carries them to a member who claims the run. Completion is applied to the job as it was picked, and tells the drain when a receipt reached a run that had already ended, for example one cancelled while its delivery was being checked. The drain then queues the end record again under the run lock, so the published record lists the pull request, and publishes it once the lock is released. A delivery job now carries the checkout, base commit and base branch it needs, and a delivering step whose run lacks them fails under the lock instead of reaching the delivery code. Red: "delivers a driver-reported URL without opening a PR, and leases the dependent in the same next", "records a pull request once, however many deliveries report it" and "records a pull request delivered after the run ended, and queues the end record again with it" failed with "expected undefined to deeply equal [ { …(5) } ]" (no deliveries on the run); the records test failed with "expected undefined to deeply equal [ …(3) ]". The handover test failed with "expected false to deeply equal" against a snapshot schema without deliveries. --- src/__tests__/workflow-delivery.test.ts | 58 +++++++++++++++++++++-- src/__tests__/workflow-records.test.ts | 14 ++---- src/workflow/engine.ts | 62 ++++++++++++++----------- src/workflow/pr-delivery.ts | 17 +++---- src/workflow/pr-url.ts | 10 ++++ src/workflow/records.ts | 18 ++----- src/workflow/snapshot.ts | 17 +++---- src/workflow/types.ts | 19 ++++---- 8 files changed, 129 insertions(+), 86 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 24cac2928..4f7d163da 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -14,7 +14,9 @@ import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; import { acquireLock, releaseLock } from '../update.js'; import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; import { GitHubProvider } from '../providers/github/index.js'; -import { buildEndRecord } from '../workflow/records.js'; +import { buildEndRecord, pendingRecords } from '../workflow/records.js'; +import { projectRun } from '../workflow/snapshot.js'; +import * as worktree from '../workflow/worktree.js'; import { cancelRun } from '../workflow/run-controls.js'; import type { LocalConfig } from '../types.js'; import type { RunState } from '../workflow/types.js'; @@ -199,7 +201,7 @@ describe('delivering a step', () => { expect(review.step).toBe('review'); const run = await state(config, runId); expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', settledAt: 2 }); - expect(run.copies?.[branch]?.pr).toEqual({ url: 'https://review.acme.internal/changes/4821', step: 'fix', pass: 1 }); + expect(run.deliveries).toEqual([{ url: 'https://review.acme.internal/changes/4821', branch, base: 'main', step: 'fix', pass: 1 }]); expect(create).not.toHaveBeenCalled(); await report(config, runId, { attempt: review.attempt, revision: review.revision, outcome: 'success' }); @@ -243,7 +245,7 @@ describe('delivering a step', () => { expect(await cancelRun(config, runId, 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'], attempts: [] }, run: { status: 'failed' } } }); const run = await state(config, runId); expect(run.steps).toMatchObject([{ id: 'fix', status: 'cancelled', attempt: fix.attempt }, { id: 'review', status: 'skipped' }]); - expect(run.copies?.[`teamai/${runId}/fix/p1`]?.pr).toBeUndefined(); + expect(run.deliveries).toBeUndefined(); }); it.each([ @@ -263,7 +265,7 @@ describe('delivering a step', () => { { id: 'review', status: 'skipped' }, ]); expect(run.steps[0]).not.toMatchObject({ failure: { message: expect.stringContaining('src/app.ts') } }); - expect(run.copies?.[`teamai/${runId}/fix/p1`]?.pr).toBeUndefined(); + expect(run.deliveries).toBeUndefined(); for (const call of calls) expect(call).not.toHaveBeenCalled(); }); @@ -289,6 +291,52 @@ describe('delivering a step', () => { for (const call of calls) expect(call).not.toHaveBeenCalled(); }); + it('records a pull request once, however many deliveries report it', async () => { + const { config, root, runId } = await fixture(` - id: a\n workspace: new\n command: node a.js\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: node b.js\n deliver: pr\n`); + const url = 'https://review.acme.internal/changes/9'; + const a = await take(config, runId); + await succeed(config, runId, a, { to: pushWork(root, a), pr_url: url }); + const b = await take(config, runId); + await succeed(config, runId, b, { to: pushWork(root, b), pr_url: url }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).deliveries).toEqual([{ url, branch: `teamai/${runId}/a/p1`, base: 'main', step: 'a', pass: 1 }]); + }); + + it('records a pull request delivered after the run ended, and queues the end record again with it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const url = 'https://review.acme.internal/changes/3'; + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: url }); + const read = worktree.fetchMissingCommits; + vi.spyOn(worktree, 'fetchMissingCommits').mockImplementation(async (dir, options) => { + await cancelRun(config, runId); + return read(dir, options); + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch: `teamai/${runId}/fix/p1`, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + }); + + it('hands over a delivering step with what it delivers, and the run with its receipts', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const url = 'https://review.acme.internal/changes/5'; + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to, pr_url: url }); + const branch = `teamai/${runId}/fix/p1`; + + const delivering = projectRun(await state(config, runId), {}); + expect(delivering.ok && delivering.value.steps[0]).toMatchObject({ status: 'delivering', delivery: { branch, head: to, reported: url } }); + await take(config, runId); + const delivered = projectRun(await state(config, runId), {}); + expect(delivered.ok && delivered.value.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + }); + it.todo('delivers a workflow step with workspace: new once, against the delivered branch of the step it needs'); it.todo('delivers each for_each item against the delivered branch of the item it waited for, and lists the PRs in item order'); it.todo('targets the nearest delivered branch, passing over a need that was skipped'); @@ -325,6 +373,6 @@ describe('reported URLs from a driver', () => { accepted: true, revision: 2, step: { id: 'write', status: 'succeeded', outcome: 'success' }, cleanup: true, dropped: [{ field: 'changes.pr_url', reason: 'the step has no deliver: pr' }], } }); - expect((await state(config, runId)).copies?.[`teamai/${runId}/write/p1`]?.pr).toBeUndefined(); + expect((await state(config, runId)).deliveries).toBeUndefined(); }); }); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index 5647e802f..4042c6f4e 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -397,24 +397,18 @@ describe('#1083 workflow run records', () => { expect(remerged.endedAt).toBe(end.endedAt); }); - it('lists the pull requests delivered on the run\'s branches once each, in step and pass order, and never a step output', () => { + it('lists the pull requests delivered on the run in the order they first appeared, and never a step output', () => { const pr = (n: number): string => `https://github.com/acme/app/pull/${n}`; - const recorded = (n: number, step: string, pass: number) => ({ pr: { url: PrUrlSchema.parse(pr(n)), step, pass } }); + const receipt = (n: number, step: string, pass: number) => ({ url: PrUrlSchema.parse(pr(n)), branch: `teamai/r1234567890abcdef/${step}/p${pass}`, base: 'main', step, pass }); const run = runState({ steps: [ { id: 'one', status: 'succeeded', attempt: 'a1', settledAt: 2, outcome: 'success', outputs: { pr_url: 'https://untrusted.example/1' } }, { id: 'two', status: 'succeeded', attempt: 'a2', settledAt: 3, outcome: 'success' }, ], - copies: { - 'teamai/r1234567890abcdef/two/p2': recorded(4, 'two', 2), - 'teamai/r1234567890abcdef/two/p1': recorded(3, 'two', 1), - 'teamai/r1234567890abcdef/one/p1': recorded(2, 'one', 1), - 'teamai/r1234567890abcdef/loop/p1': recorded(1, 'loop.write', 1), - 'teamai/r1234567890abcdef/other/p1': { commit: 'c'.repeat(40) }, - }, + deliveries: [receipt(3, 'two', 1), receipt(1, 'loop.write', 1), receipt(2, 'one', 1)], }); - expect(buildEndRecord(run, driver, 'tester').prs).toEqual([pr(1), pr(2), pr(3), pr(4)]); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual([pr(3), pr(1), pr(2)]); expect(buildEndRecord(runState(), driver, 'tester')).not.toHaveProperty('prs'); }); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index dae1aaa1f..b6a21e115 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -898,32 +898,37 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { if (run.status !== 'running') return undefined; const settledRun = reconcileRun(run); if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; - const step = run.steps.find((item): item is DeliveringStep => item.status === 'delivering'); - if (!step) return undefined; - return { - key: deliveryKey(step), runId: run.id, - ...(run.checkout ? { checkout: run.checkout } : {}), ...(run.base ? { base: run.base.commit } : {}), - ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), - }; + for (const step of run.steps) { + if (step.status !== 'delivering') continue; + const key = deliveryKey(step); + if (!run.checkout || !run.base?.ref) { + deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); + continue; + } + return { + key, runId: run.id, checkout: run.checkout, base: run.base.commit, target: run.base.ref, + ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), + }; + } + return undefined; } /** - * Applies a delivery's outcome. A pull request is recorded on its branch whatever became of the step meanwhile: one - * opened while a cancel landed cannot be taken back, so the run says it exists. The step settles only while it still - * delivers what `job` delivered, at the revision its agent's report settled it. + * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step + * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The step settles only + * while it still delivers what `job` delivered, at the revision its agent's report settled it. `requeueEnd` is set when + * the receipt reached a run that already ended, whose end record must go out again to list it. */ -function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome): void { - if (outcome.ok) { - const copies = run.copies ??= {}; - const copy = copies[job.key.branch] ?? {}; - copy.pr ??= { url: outcome.pr, step: job.key.step, pass: job.key.pass }; - copies[job.key.branch] = copy; +function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome): { requeueEnd: boolean } { + const receipted = outcome.ok && !run.deliveries?.some((receipt) => receipt.url === outcome.pr); + if (outcome.ok && receipted) { + (run.deliveries ??= []).push({ url: outcome.pr, branch: job.key.branch, base: job.target, step: job.key.step, pass: job.key.pass }); } if (run.status === 'running') reconcileRun(run); const step = deliveringStep(run, job.key); - if (!step) return; - if (outcome.ok) delivered(step); - else deliveryFailed(step, deliveryFailure(job, outcome.fault)); + if (step && outcome.ok) delivered(step); + else if (step && !outcome.ok) deliveryFailed(step, deliveryFailure(job, outcome.fault)); + return { requeueEnd: receipted && run.status !== 'running' }; } /** @@ -935,25 +940,30 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise step.status === 'delivering')) return; - await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { + const drained = await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { + let requeued = false; for (;;) { const picked = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); const job = nextDelivery(run); - return { value: job, ...(job && JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; + return { value: job, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); - if (picked.status !== 'done' || !picked.value) return; + if (picked.status !== 'done' || !picked.value) return requeued; const job = picked.value; const outcome = await deliverPr(job); const applied = await updateRun(config, runId, async (run) => { - if (run === 'missing') return { value: false }; - completeDelivery(run, job, outcome); - return { value: true, write: { ...run, updatedAt: new Date().toISOString() } }; + if (run === 'missing') return { value: undefined }; + const { requeueEnd } = completeDelivery(run, job, outcome); + // Queued under the lock with the receipt, as finishRun queues an end; it publishes once the lock is released. + if (requeueEnd) run.endRecordQueued = await queueRunRecord(config, run, run.driver, 'end'); + return { value: requeueEnd, write: { ...run, updatedAt: new Date().toISOString() } }; }); - if (applied.status !== 'done' || !applied.value) return; + if (applied.status !== 'done' || applied.value === undefined) return requeued; + requeued ||= applied.value; } }); + if (drained.held && drained.value) await publishRunRecords(config); } export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index be6673f69..cc81f7df1 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -21,14 +21,16 @@ export interface DeliveryJob { key: DeliveryKey; runId: string; /** The checkout the run started or was claimed in, which reads the delivered branch. */ - checkout?: string; + checkout: string; /** The run base commit the delivered branch is compared with. */ - base?: string; + base: string; + /** The branch the pull request goes into. */ + target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ reported?: PrUrl; } -/** Why a delivery opened no pull request. */ +/** Why a delivery opened no pull request. `no_checkout` is found under the run lock, before there is a job. */ export type DeliveryFault = | { kind: 'no_checkout' } | { kind: 'unreadable' } @@ -55,7 +57,6 @@ export async function deliverPr(job: DeliveryJob): Promise { /** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promise<{ files: string[] } | { fault: DeliveryFault }> { - if (!checkout || !base) return { fault: { kind: 'no_checkout' } }; if (!OBJECT_ID.test(key.head) || !OBJECT_ID.test(base)) return { fault: { kind: 'unreadable' } }; const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [base, key.head] }); if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; @@ -67,14 +68,14 @@ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promis const LISTED_FILES = 20; /** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ -export function deliveryFailure(job: DeliveryJob, fault: DeliveryFault): WorkflowFailure { - const { step, branch } = job.key; - const retry = `teamai workflow retry ${job.runId} ${step}`; +export function deliveryFailure({ runId, key }: Pick, fault: DeliveryFault): WorkflowFailure { + const { step, branch } = key; + const retry = `teamai workflow retry ${runId} ${step}`; switch (fault.kind) { case 'no_checkout': return { code: 'delivery_failed', - message: `PR delivery for step "${step}" failed: the run records no checkout and base to compare branch ${branch} with, so its changes cannot be checked. The work is kept in branch ${branch}. Open a pull request from it yourself, or start the run again from a checkout of its repository.`, + message: `PR delivery for step "${step}" failed: the run records no checkout, or no base branch, to deliver branch ${branch} from and into. The work is kept in branch ${branch}. Open a pull request from it yourself, or start the run again from a checkout of its repository.`, }; case 'unreadable': return { diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts index 53e47dd22..67e967875 100644 --- a/src/workflow/pr-url.ts +++ b/src/workflow/pr-url.ts @@ -38,3 +38,13 @@ export function sanitizePrUrl(value: unknown): { url: PrUrl; reason?: never } | const parsed = PrUrlSchema.safeParse(value); return parsed.success ? { url: parsed.data } : { reason: parsed.error.issues[0]?.message ?? 'is not a valid URL' }; } + +/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ +export const DeliveryReceiptSchema = z.object({ + url: PrUrlSchema, + branch: z.string(), + base: z.string(), + step: z.string(), + pass: z.number().int(), +}); +export type DeliveryReceipt = z.infer; diff --git a/src/workflow/records.ts b/src/workflow/records.ts index 5e7b334ca..2381e48a0 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -97,7 +97,7 @@ export type WorkflowRunRecord = { stepCounts?: { total: number; succeeded: number; failed: number; skipped: number; cancelled: number }; /** Settled attempts across the run — the cumulative iteration count. */ iterations?: number; - /** The pull requests the run delivered, in step order. Public on a public team repo; absent when it delivered none. */ + /** The pull requests the run delivered, in the order they first appeared. Public on a public team repo; absent when it delivered none. */ prs?: string[]; username: string; publishedAt: string; @@ -170,19 +170,6 @@ function failureOf(run: RunState): RecordedFailure | undefined { return [first, ...failed.filter(under)].map((step) => step.failure).find(isRecordedFailure)?.code; } -/** - * The pull requests the run's steps delivered, once each, in the order of the steps that recorded them and, for one - * step, of its passes. That is item order for a `for_each`, whichever item finished first, and a loop that revised one - * branch lists its pull request once. A step a loop removed from the run lists first. Only what delivery recorded on a - * branch is listed, never a step's outputs. - */ -function prsOf(run: RunState): string[] { - const position = new Map(run.steps.map((step, index) => [step.id, index])); - const recorded = Object.values(run.copies ?? {}).flatMap((copy) => (copy.pr ? [copy.pr] : [])); - const ordered = recorded.sort((left, right) => (position.get(left.step) ?? -1) - (position.get(right.step) ?? -1) || left.pass - right.pass); - return [...new Set(ordered.map((pr) => pr.url))]; -} - /** * Build the end-phase record: outcome, duration, iterations, and step counts, * all derived as counts so no step payload crosses into the record. @@ -225,7 +212,8 @@ export function buildEndRecord( ]), ...(run.archive ?? []).map((archived) => entry(archived.step, archived.pass, archived)), ]; - const prs = prsOf(run); + // Receipts are what delivery recorded, never a step's outputs, and each URL appears once. + const prs = (run.deliveries ?? []).map((receipt) => receipt.url); return { ...buildStartRecord(run, driver, username, now), phase: 'end', diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index e9a104116..2f9c9718a 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { canonicalJson } from '../utils/canonical-json.js'; import { redact } from '../utils/redact.js'; import { isRecord } from './expression.js'; -import { PrUrlSchema } from './pr-url.js'; +import { DeliveryReceiptSchema } from './pr-url.js'; import { ArchivedPassSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; @@ -53,14 +53,9 @@ const BodySchema = z.object({ archive: z.array(withCommitIds(ArchivedPassSchema)), round: z.number().int().optional(), base: z.object({ ref: z.string().optional(), commit: CommitIdSchema }).optional(), - /** - * The run's workspaces by branch, with the commit each was last reported at and the pull request it was delivered to. - * Which driver holds one stays local. - */ - copies: z.record(z.object({ - commit: CommitIdSchema.optional(), - pr: z.object({ url: PrUrlSchema, step: z.string(), pass: z.number().int() }).optional(), - })), + /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ + copies: z.record(z.object({ commit: CommitIdSchema.optional() })), + deliveries: z.array(DeliveryReceiptSchema).optional(), sessionContinuations: z.array(SessionContinuationSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ scorers: z.record(z.object({ sourceHash: z.string() })).optional(), @@ -198,7 +193,7 @@ const RUN_RULES = { end: 'publish', failure: 'rewrite', inputs: 'publish', sourceHash: 'publish', teamRevision: 'publish', definition: 'rewrite', workflows: 'rewrite', scorers: 'rewrite', steps: 'rewrite', settledAttempts: 'rewrite', settledCount: 'rewrite', endRecordQueued: 'drop', cancelledAttempts: 'rewrite', archive: 'rewrite', round: 'publish', base: 'publish', - checkout: 'drop', copies: 'rewrite', sessionContinuations: 'publish', publication: 'drop', createdAt: 'publish', updatedAt: 'publish', + checkout: 'drop', copies: 'rewrite', deliveries: 'publish', sessionContinuations: 'publish', publication: 'drop', createdAt: 'publish', updatedAt: 'publish', } satisfies FieldRules; const STEP_RULES = { @@ -227,7 +222,7 @@ const FAILURE_RULES = { 'code': 'publish', message: 'rewrite' } satisfies FieldR // A lease another machine held is released when the run is handed over, so the session it offered stays here. const LEASE_RULES = { attempt: 'publish', revision: 'publish', driver: 'rewrite', session: 'drop' } satisfies FieldRules; -const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish', pr: 'publish' } satisfies FieldRules[string]>; +const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish' } satisfies FieldRules[string]>; const FAILURE_REWRITES = { message: () => ({ message: HANDED_FAILURE_MESSAGE }), diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 77f231273..1171ef134 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,17 +1,10 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; -import type { PrUrl } from './pr-url.js'; +import type { DeliveryReceipt } from './pr-url.js'; import type { ArchivedPass, SessionContinuation, StepState } from './step-state.js'; import type { LeaseSession } from './workspace.js'; -/** The pull request a branch was delivered to, and the step pass whose delivery recorded it. */ -export interface BranchPr { - url: PrUrl; - step: string; - pass: number; -} - /** The driver's lease ownership, independent of its per-request capabilities. */ export interface DriverIdentity { kind: string; @@ -264,10 +257,14 @@ export interface RunState { checkout?: string; /** * Workspaces this run has handed out, by branch. A `continue` step stays with the driver that opened the copy. A copy a - * claim brought from another machine has no owner until a driver leases it. `pr` is the pull request delivered for the - * branch, written once by the step whose delivery first recorded it: one branch has one pull request. + * claim brought from another machine has no owner until a driver leases it. + */ + copies?: Record; + /** + * The pull requests delivered on this run, once each, in the order they first appeared. Append-only: a receipt says a + * request was opened, never that it is still open, so whether a delivery reuses one is its host's lookup to say. */ - copies?: Record; + deliveries?: DeliveryReceipt[]; /** * Sessions a step resumed and continued under another id, in report order: a copy on another machine or in another * tool. `to` holds the turns of `from` and the ones that step added, so a step that would resume `from` resumes `to`. From e7871bbfa1179a7b87d8e16a1cfcb936f12242c7 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:18:23 +0200 Subject: [PATCH 10/77] feat(workflow): open the pull request through the origin's provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A step with deliver: pr whose driver reported no URL failed delivery even when the origin was on GitHub. Delivery now resolves the provider from the checkout's configured remote.origin.url. It looks up the open request of the delivered branch into the run base branch, and opens one only when there is none. A create that fails is looked up again, since the host may have opened the request before it failed or timed out. Right before the create, the run is asked under its lock whether the step still delivers this job. If it does, the step is marked `creating` in the same write, so a request opened by a process that then stopped can be found rather than opened twice. A cancelled step keeps its delivery for the same reason. The create runs outside the member checkout and names the head branch, so nothing is pushed from it. The lookup never prompts or logs in. No provider error text is stored, because it may hold a credential. Providers expose a read-only `pullRequests` lookup with a single `findOpen` member. GitHub implements it. Generic git and the hosts without a lookup yet are null, and delivery there fails with delivery_failed, naming the branch kept on origin and the retry command. The per-provider workflow-pr modules are removed. A structural test fails if any provider file or the delivery code gains a merge, accept or auto-merge call, and pins the lookup's key set. Red: "opens one pull request from the pushed branch into the run base branch", both "records the pull request ... rather than opening another" cases, "keeps the work on origin when GitHub cannot be reached" and "drops a URL with credentials" failed with `no lease: {"data":{"done":{...,"outcome": "failed"}}}`, since delivery failed for want of a provider; "asks the run before it opens a pull request" failed with `expected { ok: false, …(1) } to deeply equal { ok: false, …(1) }`; "cancels a delivering step" failed with `expected [ { id: 'fix', …(4) }, …(1) ] to match object [ { id: 'fix', …(3) }, …(1) ]`; the GitHub lookup tests failed with "GitHub has no pull request lookup", the origin tests with "(0 , providerForRemote) is not a function", and the generic git test with "expected undefined to be null". The merge guard failed listing gh-cli.ts against two patterns with a local `ghExec(['pr', 'merge', n, '--squash'])`, and its key-set check failed tsc with TS2344 "Expected: literal string: findOpen, Actual: literal string: merge" with a local `merge?` member. --- src/__tests__/helpers/fake-gh.ts | 79 +++++++++++ src/__tests__/no-provider-merge.test.ts | 43 ++++++ src/__tests__/workflow-delivery.test.ts | 142 ++++++++++++++++---- src/__tests__/workflow-pr-providers.test.ts | 141 +++++++++---------- src/providers/cnb/index.ts | 5 +- src/providers/cnb/workflow-pr.ts | 41 ------ src/providers/git/index.ts | 2 + src/providers/gitcode/index.ts | 5 +- src/providers/gitcode/workflow-pr.ts | 37 ----- src/providers/github/gh-cli.ts | 51 +++++++ src/providers/github/index.ts | 13 +- src/providers/github/workflow-pr.ts | 48 ------- src/providers/gitlab/index.ts | 11 +- src/providers/gitlab/workflow-pr.ts | 42 ------ src/providers/registry.ts | 49 ++++++- src/providers/tgit/index.ts | 6 +- src/providers/tgit/workflow-pr.ts | 32 ----- src/providers/types.ts | 29 +++- src/workflow/engine.ts | 21 ++- src/workflow/pr-delivery.ts | 93 ++++++++++++- src/workflow/step-state.ts | 21 ++- 21 files changed, 559 insertions(+), 352 deletions(-) create mode 100644 src/__tests__/helpers/fake-gh.ts create mode 100644 src/__tests__/no-provider-merge.test.ts delete mode 100644 src/providers/cnb/workflow-pr.ts delete mode 100644 src/providers/gitcode/workflow-pr.ts delete mode 100644 src/providers/github/workflow-pr.ts delete mode 100644 src/providers/gitlab/workflow-pr.ts delete mode 100644 src/providers/tgit/workflow-pr.ts diff --git a/src/__tests__/helpers/fake-gh.ts b/src/__tests__/helpers/fake-gh.ts new file mode 100644 index 000000000..110ed27d5 --- /dev/null +++ b/src/__tests__/helpers/fake-gh.ts @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { vi } from 'vitest'; +import { z } from 'zod'; +import * as cliPath from '../../utils/cli-path.js'; + +const CallSchema = z.object({ args: z.array(z.string()), cwd: z.string(), prompt: z.string().optional() }); +export type GhCall = z.infer; + +/** + * How the fake answers. `lookup: 'fail'` fails every `gh api`. `create: 'fail'` fails `gh pr create` and opens nothing; + * `'fail-after-open'` opens the request and still fails, as a create that timed out after GitHub took it. `open` + * replaces the open requests. + */ +export interface FakeGhOptions { + lookup?: 'fail'; + create?: 'fail' | 'fail-after-open'; + open?: unknown[]; +} + +function script(dir: string): string { + return `#!${process.execPath} +const fs = require('node:fs'); +const path = require('node:path'); +const dir = ${JSON.stringify(dir)}; +const read = (file, fallback) => { try { return JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')); } catch { return fallback; } }; +const args = process.argv.slice(2); +const options = read('options.json', {}); +fs.appendFileSync(path.join(dir, 'calls.jsonl'), JSON.stringify({ args, cwd: process.cwd(), prompt: process.env.GIT_TERMINAL_PROMPT }) + '\\n'); +const open = read('open.json', []); +if (args[0] === 'api') { + if (options.lookup === 'fail') { console.error('HTTP 401: Bad credentials for token ghp_fake_secret'); process.exit(1); } + console.log(JSON.stringify(open)); + process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'create') { + const value = (flag) => args[args.indexOf(flag) + 1]; + const repo = value('-R'); + const pull = { state: 'open', html_url: 'https://github.com/' + repo + '/pull/' + (open.length + 1), head: { ref: value('-H'), repo: { full_name: repo } }, base: { ref: value('-B'), repo: { full_name: repo } } }; + if (options.create !== 'fail') fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify([...open, pull])); + if (options.create) { console.error('pull request create failed for token ghp_fake_secret'); process.exit(1); } + console.log(pull.html_url); + process.exit(0); +} +console.error('unexpected gh call: ' + args.join(' ')); +process.exit(9); +`; +} + +/** + * A `gh` that keeps open pull requests under `dir` and logs each call with its cwd and GIT_TERMINAL_PROMPT. It answers + * `gh api` with every open request and `gh pr create` with a new one, and its failures print a token, so a test can + * show none is kept. teamai resolves `gh` to it until the test's mocks are restored. + */ +export function fakeGh(dir: string, options: FakeGhOptions = {}): { configure(next: FakeGhOptions): void; calls(): GhCall[] } { + fs.mkdirSync(dir, { recursive: true }); + const bin = path.join(dir, 'gh'); + fs.writeFileSync(bin, script(dir), { mode: 0o755 }); + const fake = { + configure(next: FakeGhOptions): void { + fs.writeFileSync(path.join(dir, 'options.json'), JSON.stringify(next)); + if (next.open) fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify(next.open)); + }, + calls(): GhCall[] { + const log = path.join(dir, 'calls.jsonl'); + if (!fs.existsSync(log)) return []; + return fs.readFileSync(log, 'utf8').trim().split('\n').map((line) => CallSchema.parse(JSON.parse(line))); + }, + }; + fake.configure(options); + resolveGh(bin); + return fake; +} + +/** Makes teamai find `gh` at `bin`, or find none, whatever this machine has installed. */ +export function resolveGh(bin: string | null): void { + const resolve = cliPath.resolveCliPath; + vi.spyOn(cliPath, 'resolveCliPath').mockImplementation((cmd, platform) => (cmd === 'gh' ? bin : resolve(cmd, platform))); +} diff --git a/src/__tests__/no-provider-merge.test.ts b/src/__tests__/no-provider-merge.test.ts new file mode 100644 index 000000000..384c3d9c7 --- /dev/null +++ b/src/__tests__/no-provider-merge.test.ts @@ -0,0 +1,43 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, expectTypeOf, it } from 'vitest'; +import type { PullRequestLookup } from '../providers/types.js'; + +const SRC = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +/** Every provider file, a new adapter included, and the delivery code that calls them. */ +function scope(): string[] { + const providers = fs.readdirSync(path.join(SRC, 'providers'), { recursive: true, encoding: 'utf8' }) + .filter((file) => file.endsWith('.ts')) + .map((file) => path.join('providers', file)); + return [...providers, path.join('workflow', 'pr-delivery.ts')]; +} + +const MERGE_CALLS = [ + /['"`](?:pr|mr)['"`]\s*,\s*['"`]merge/i, + /\b(?:pr|mr)\s+merge\b/i, + /['"`]merge(?:[-_]?pulls?)?['"`]/i, + // A REST merge or accept endpoint, not the words "pull/merge request". + /\/merge\b(?!\s+requests?)/i, + /\/accept\b/i, + /auto_?merge|merge_when_pipeline_succeeds|mergePullRequest/i, +]; + +describe('workflow delivery never merges', () => { + it('no provider file or delivery code holds a merge, accept or auto-merge call', () => { + const found = scope().flatMap((file) => { + const text = fs.readFileSync(path.join(SRC, file), 'utf8'); + return MERGE_CALLS.filter((pattern) => pattern.test(text)).map((pattern) => `${file}: ${pattern}`); + }); + expect(found).toEqual([]); + }); + + it('the lookup delivery reads providers through can only find an open request', () => { + expectTypeOf().toEqualTypeOf<'findOpen'>(); + }); + + it('delivery code imports nothing from the engine, which applies its outcome', () => { + expect(fs.readFileSync(path.join(SRC, 'workflow', 'pr-delivery.ts'), 'utf8')).not.toMatch(/from '\.\/engine\.js'/); + }); +}); diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 4f7d163da..87539b233 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -13,11 +13,12 @@ import { deliveryLockPath, readRun } from '../workflow/store.js'; import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; import { acquireLock, releaseLock } from '../update.js'; import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; -import { GitHubProvider } from '../providers/github/index.js'; +import { fakeGh } from './helpers/fake-gh.js'; +import { deliverPr } from '../workflow/pr-delivery.js'; import { buildEndRecord, pendingRecords } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; import * as worktree from '../workflow/worktree.js'; -import { cancelRun } from '../workflow/run-controls.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; import type { LocalConfig } from '../types.js'; import type { RunState } from '../workflow/types.js'; @@ -26,6 +27,7 @@ _setDriverPollMs(20); const dirs: string[] = []; afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); @@ -77,7 +79,7 @@ const head = 'c'.repeat(40); /** Takes the next lease as a driver that speaks only CLI JSON does. */ async function take(config: LocalConfig, runId: string): Promise { const next = await leaseNext(config, runId, remote, writable); - if (!('data' in next)) throw new Error(JSON.stringify(next)); + if (!('data' in next) || !next.data.lease) throw new Error(`no lease: ${JSON.stringify(next)}`); return LeaseSchema.parse(next.data.lease); } @@ -112,13 +114,15 @@ async function succeed(config: LocalConfig, runId: string, lease: Lease, changes ...(lease.workspace ? { changes: { branch: lease.workspace.branch, base, from: base, to, commits: [to], ...changes } } : {}) }); } -/** The pull request lookups and creates a GitHub origin could see, none of which may run. */ -function provider() { - return [ - vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockRejectedValue(new Error('no PR may be opened')), - vi.spyOn(GitHubProvider.prototype, 'findOpenPullRequest').mockRejectedValue(new Error('no PR may be looked up')), - vi.spyOn(GitHubProvider.prototype, 'validatePullRequest').mockRejectedValue(new Error('no PR may be read')), - ]; +/** The value that follows `flag` in a gh call. */ +function flag(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at < 0 ? undefined : args[at + 1]; +} + +/** An open pull request as GitHub lists it. */ +function pull(number: number, head: string, base = 'main', repo = 'fixture/app') { + return { state: 'open', html_url: `https://github.com/${repo}/pull/${number}`, head: { ref: head, repo: { full_name: repo } }, base: { ref: base, repo: { full_name: repo } } }; } const FIX_THEN_REVIEW = ` - id: fix\n workspace: new\n command: node fix.js\n deliver: pr\n - id: review\n needs: [fix]\n command: echo review\n`; @@ -180,8 +184,8 @@ describe('workflow PR delivery contract', () => { describe('delivering a step', () => { it('delivers a driver-reported URL without opening a PR, and leases the dependent in the same next', async () => { - const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockRejectedValue(new Error('no PR may be opened')); const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); const fix = await take(config, runId); const branch = `teamai/${runId}/fix/p1`; expect(fix).toMatchObject({ step: 'fix', workspace: { branch } }); @@ -202,7 +206,7 @@ describe('delivering a step', () => { const run = await state(config, runId); expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', settledAt: 2 }); expect(run.deliveries).toEqual([{ url: 'https://review.acme.internal/changes/4821', branch, base: 'main', step: 'fix', pass: 1 }]); - expect(create).not.toHaveBeenCalled(); + expect(gh.calls()).toEqual([]); await report(config, runId, { attempt: review.attempt, revision: review.revision, outcome: 'success' }); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); @@ -221,9 +225,14 @@ describe('delivering a step', () => { await released; }); - it('fails a delivery that has no URL and no host to open one on, naming the branch that keeps the work', async () => { + it.each([ + ['a filesystem path', (root: string) => path.join(root, 'remote.git')], + ['a host only git reaches', () => 'https://git.example.internal/fixture/app.git'], + ])('fails a delivery that has no URL and an origin at %s, naming the branch that keeps the work', async (_case, origin) => { const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); - git(cwd, ['remote', 'set-url', 'origin', path.join(root, 'remote.git')]); + const gh = fakeGh(path.join(root, 'gh')); + git(cwd, ['remote', 'set-url', 'origin', origin(root)]); + git(cwd, ['config', `url.${path.join(root, 'remote.git')}.insteadOf`, 'https://git.example.internal/fixture/app.git']); const fix = await take(config, runId); const to = pushWork(root, fix); expect(await succeed(config, runId, fix, { to })).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); @@ -235,6 +244,84 @@ describe('delivering a step', () => { { id: 'review', status: 'skipped' }, ]); expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(gh.calls()).toEqual([]); + }); + + it('opens one pull request from the pushed branch into the run base branch, outside the member checkout, and records it', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + expect(await succeed(config, runId, fix, { to })).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries).toEqual([{ url: 'https://github.com/fixture/app/pull/1', branch, base: 'main', step: 'fix', pass: 1 }]); + const [lookup, create, ...rest] = gh.calls(); + expect(rest).toEqual([]); + expect(lookup?.args[0]).toBe('api'); + expect(lookup?.prompt).toBe('0'); + const query = new URLSearchParams(lookup?.args[1]?.split('?')[1]); + expect([lookup?.args[1]?.split('?')[0], query.get('state'), query.get('head'), query.get('base')]).toEqual(['repos/fixture/app/pulls', 'open', `fixture:${branch}`, 'main']); + expect(create?.args.slice(0, 2)).toEqual(['pr', 'create']); + expect([flag(create?.args ?? [], '-R'), flag(create?.args ?? [], '-H'), flag(create?.args ?? [], '-B')]).toEqual(['fixture/app', branch, 'main']); + expect(fs.realpathSync(create?.cwd ?? cwd).startsWith(fs.realpathSync(root))).toBe(false); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + }); + + it.each([ + ['a delivery that stopped after opening it', {}], + ['a create that failed after GitHub took it', { create: 'fail-after-open' as const }], + ])('records the pull request %s rather than opening another', async (_case, options) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { ...options, open: 'create' in options ? [] : [pull(5, `${branch}-other`), pull(6, branch, 'main', 'fork/app'), pull(7, branch)] }); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const url = 'create' in options ? 'https://github.com/fixture/app/pull/1' : 'https://github.com/fixture/app/pull/7'; + const run = await state(config, runId); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(gh.calls().map((call) => call.args[0])).toEqual('create' in options ? ['api', 'pr', 'api'] : ['api']); + // A step marked before it asks GitHub to open a request looks one up first after a crash; one found needs no mark. + expect(run.steps[0]).toMatchObject({ delivery: { branch, ...('create' in options ? { creating: true } : {}) } }); + if (!('create' in options)) expect(run.steps[0]).not.toHaveProperty('delivery.creating'); + const handed = projectRun(run, {}); + expect(handed.ok && handed.value.steps[0]).toEqual(run.steps[0]); + }); + + it('keeps the work on origin when GitHub cannot be reached, stores none of its output, and retry delivers without running the step again', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }); + expect(failed.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(JSON.stringify(failed)).not.toContain('ghp_fake_secret'); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + + gh.configure({}); + expect(await retryStep(config, runId, 'fix')).toMatchObject({ data: {} }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('asks the run before it opens a pull request, and opens none for a step that stopped delivering', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + if (!fix.workspace) throw new Error('fix has no workspace'); + const to = pushWork(root, fix); + const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main' }; + + expect(await deliverPr(job, async () => false)).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); + expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); }); it('cancels a delivering step: it delivers nothing and its dependents do not run', async () => { @@ -244,7 +331,9 @@ describe('delivering a step', () => { expect(await cancelRun(config, runId, 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'], attempts: [] }, run: { status: 'failed' } } }); const run = await state(config, runId); - expect(run.steps).toMatchObject([{ id: 'fix', status: 'cancelled', attempt: fix.attempt }, { id: 'review', status: 'skipped' }]); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'cancelled', attempt: fix.attempt, delivery: { branch: `teamai/${runId}/fix/p1`, head } }, { id: 'review', status: 'skipped' }, + ]); expect(run.deliveries).toBeUndefined(); }); @@ -252,8 +341,8 @@ describe('delivering a step', () => { ['TeamAI would open the pull request', {}], ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], ])('refuses a branch that changes protected paths, naming them, before any provider request, when %s', async (_case, changes) => { - const calls = provider(); const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); const fix = await take(config, runId); const to = pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n', 'src/app.ts': 'ok\n', 'packages/web/CLAUDE.md': 'rules\n' }); await succeed(config, runId, fix, { to, ...changes }); @@ -266,7 +355,7 @@ describe('delivering a step', () => { ]); expect(run.steps[0]).not.toMatchObject({ failure: { message: expect.stringContaining('src/app.ts') } }); expect(run.deliveries).toBeUndefined(); - for (const call of calls) expect(call).not.toHaveBeenCalled(); + expect(gh.calls()).toEqual([]); }); it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { @@ -282,13 +371,13 @@ describe('delivering a step', () => { }); it('refuses protected changes the branch inherited from a step it needs, though the delivering step left them alone', async () => { - const calls = provider(); const write = (file: string) => `node -e "require('fs').mkdirSync(require('path').dirname('${file}'),{recursive:true});require('fs').writeFileSync('${file}','x')"`; - const { config, cwd, runId } = await fixture(` - id: earlier\n workspace: new\n command: ${write('.github/workflows/ci.yml')}\n - id: later\n needs: [earlier]\n workspace: new\n command: ${write('change.txt')}\n deliver: pr\n`); + const { config, cwd, root, runId } = await fixture(` - id: earlier\n workspace: new\n command: ${write('.github/workflows/ci.yml')}\n - id: later\n needs: [earlier]\n workspace: new\n command: ${write('change.txt')}\n deliver: pr\n`); + const gh = fakeGh(path.join(root, 'gh')); expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); expect((await state(config, runId)).steps[1]).toMatchObject({ id: 'later', status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('.github/workflows/ci.yml') } }); - for (const call of calls) expect(call).not.toHaveBeenCalled(); + expect(gh.calls()).toEqual([]); }); it('records a pull request once, however many deliveries report it', async () => { @@ -353,18 +442,19 @@ describe('protected paths', () => { describe('reported URLs from a driver', () => { it('drops a URL with credentials without failing the report, and never stores, echoes or publishes it', async () => { - const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest'); - const { config, runId } = await fixture(FIX_THEN_REVIEW); - const accepted = await succeed(config, runId, await take(config, runId), { pr_url: 'https://user:s3cret-value@github.com/fixture/app/pull/1', extra: 'kept-out' }); + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://user:s3cret-value@github.com/fixture/app/pull/1', extra: 'kept-out' }); expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'fix', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }], ignored: ['changes.extra'], } }); - expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); const run = await state(config, runId); expect(run.steps[0]).not.toHaveProperty('delivery.reported'); - expect(JSON.stringify([accepted, run, buildEndRecord(run, remote, 'tester')])).not.toMatch(/s3cret-value|kept-out/); - expect(validate).not.toHaveBeenCalled(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + expect(JSON.stringify([accepted, run, buildEndRecord(run, remote, 'tester'), gh.calls()])).not.toMatch(/s3cret-value|kept-out/); }); it('drops a URL from a step that does not deliver, saying why, and still accepts the report', async () => { diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 311b48532..3c0ed19d6 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -1,97 +1,82 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import * as github from '../providers/github/gh-cli.js'; -import * as gitcode from '../providers/gitcode/gitcode-api.js'; -import * as tgit from '../providers/tgit/rest-auth.js'; -import * as cnb from '../providers/cnb/cnb-cli.js'; import { GitHubProvider } from '../providers/github/index.js'; -import { GitLabProvider } from '../providers/gitlab/index.js'; -import { GitCodeProvider } from '../providers/gitcode/index.js'; -import { TGitProvider } from '../providers/tgit/index.js'; -import { CNBProvider } from '../providers/cnb/index.js'; -import type { PrCreateOptions } from '../providers/types.js'; +import { GenericGitProvider } from '../providers/git/index.js'; +import { providerForRemote } from '../providers/registry.js'; +import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; +import { fakeGh, resolveGh } from './helpers/fake-gh.js'; -const opts: PrCreateOptions = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', title: 'Change', spawnTimeoutMs: 1000 }; -afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); +const ref: PullRequestRef = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', timeoutMs: 1000 }; +const pull = (head: string, repo = 'acme/app', number = 7) => ({ + state: 'open', html_url: `https://github.com/acme/app/pull/${number}`, head: { ref: head, repo: { full_name: repo } }, base: { ref: 'main', repo: { full_name: 'acme/app' } }, +}); + +const dirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); -describe('workflow provider PR reuse and validation', () => { - it('reuses the exact GitHub repository/head/base and never accepts a fork with the same branch name', async () => { - vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); +function lookup(): PullRequestLookup { + const found = new GitHubProvider().pullRequests; + if (!found) throw new Error('GitHub has no pull request lookup'); + return found; +} + +describe('GitHub pull request lookup', () => { + it('finds the open request of this repository branch into the target, never a fork with the same branch name', async () => { + resolveGh(null); vi.stubEnv('GITHUB_TOKEN', 'test-token'); - const pull = { state: 'open', html_url: 'https://github.com/acme/app/pull/7', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: 'main', repo: { full_name: opts.repo } } }; - const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ ...pull, head: { ref: opts.source, repo: { full_name: 'fork/app' } }, html_url: 'https://github.com/acme/app/pull/6' }, pull]))); + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([pull(ref.source, 'fork/app', 6), pull('other'), pull(ref.source)]))); vi.stubGlobal('fetch', fetcher); - const provider = new GitHubProvider(); - expect(await provider.findOpenPullRequest(opts)).toBe(pull.html_url); - const url = new URL(fetcher.mock.calls[0]![0] as string); - expect(url.searchParams.get('head')).toBe(`acme:${opts.source}`); - expect(url.searchParams.get('base')).toBe('main'); - fetcher.mockResolvedValueOnce(new Response(JSON.stringify(pull))); - expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(true); - fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...pull, base: { ...pull.base, ref: 'other' } }))); - expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(false); - expect(await provider.validatePullRequest('https://github.com/other/app/pull/7', opts)).toBe(false); - const calls = fetcher.mock.calls.length; - expect(await provider.validatePullRequest('https://private-review.example/1', opts)).toBeUndefined(); - expect(fetcher).toHaveBeenCalledTimes(calls); + + expect(await lookup().findOpen(ref)).toBe('https://github.com/acme/app/pull/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([url.pathname, url.searchParams.get('state'), url.searchParams.get('head'), url.searchParams.get('base')]) + .toEqual(['/repos/acme/app/pulls', 'open', `acme:${ref.source}`, 'main']); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify([pull(ref.source, 'fork/app', 6)]))); + expect(await lookup().findOpen(ref)).toBeUndefined(); }); - it('uses an already authenticated gh without invoking login', async () => { - vi.spyOn(github, 'isGhInstalled').mockReturnValue(true); - const cli = vi.spyOn(github, 'ghExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([{ state: 'open', html_url: 'https://github.com/acme/app/pull/8', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }]) }); - expect(await new GitHubProvider().findOpenPullRequest(opts)).toBe('https://github.com/acme/app/pull/8'); - expect(cli.mock.calls[0]?.[0][0]).toBe('api'); - expect(cli.mock.calls.flatMap(([args]) => args)).not.toContain('login'); + it('asks an installed gh with prompts off, and never logs in', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-fake-gh-')); + dirs.push(dir); + const gh = fakeGh(dir, { open: [pull(ref.source)] }); + + expect(await lookup().findOpen(ref)).toBe('https://github.com/acme/app/pull/7'); + expect(gh.calls()).toMatchObject([{ args: ['api', expect.stringMatching(/^repos\/acme\/app\/pulls\?/)], prompt: '0' }]); }); - it('fails noninteractively without GitHub credentials, without echoing response bodies', async () => { - vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); + it('fails, rather than finding nothing, without credentials or on an answer it cannot read, and never repeats what GitHub said', async () => { + resolveGh(null); vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); - await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub authentication unavailable'); + await expect(lookup().findOpen(ref)).rejects.toThrow('GitHub credentials are unavailable'); vi.stubEnv('GITHUB_TOKEN', 'test-token'); - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('private-token-in-body', { status: 403 }))); - await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub PR lookup failed (403). Check token and repository access.'); - }); - - it('validates GitLab project and branches at the configured instance, including a subpath', async () => { - vi.stubEnv('GITLAB_TOKEN', 'test-token'); vi.stubEnv('GITLAB_URL', 'https://gitlab.example/team'); - const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 5, target_project_id: 5, web_url: 'https://gitlab.example/team/acme/app/-/merge_requests/9' }; - const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([mr]))); - vi.stubGlobal('fetch', fetcher); - const provider = new GitLabProvider(); - expect(await provider.findOpenPullRequest(opts)).toBe(mr.web_url); - expect(String(fetcher.mock.calls[0]?.[0])).toContain('/team/api/v4/projects/acme%2Fapp/merge_requests?'); - fetcher.mockResolvedValueOnce(new Response(JSON.stringify(mr))); - expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(true); - expect(await provider.validatePullRequest('https://gitlab.example/team/other/app/-/merge_requests/9', opts)).toBe(false); - fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...mr, source_project_id: 99 }))); - expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(false); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('private-token-in-body', { status: 403 }))); + await expect(lookup().findOpen(ref)).rejects.toThrow(/^GitHub answered the open pull request lookup with HTTP 403\.(?!.*private-token-in-body)/); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ message: 'private-token-in-body' })))); + await expect(lookup().findOpen(ref)).rejects.toThrow(/^GitHub answered the open pull request lookup with data TeamAI cannot read\.$/); }); +}); - it('reuses TGit requests through its existing REST authentication', async () => { - const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 4, target_project_id: 4, web_url: 'https://git.woa.com/acme/app/merge_requests/3' }; - const fetcher = vi.spyOn(tgit, 'tgitFetch').mockResolvedValue(new Response(JSON.stringify([mr]))); - expect(await new TGitProvider().findOpenPullRequest(opts)).toBe(mr.web_url); - fetcher.mockResolvedValueOnce(new Response(JSON.stringify([mr]))); - expect(await new TGitProvider().validatePullRequest(mr.web_url, opts)).toBe(true); +describe('the provider of a checkout origin', () => { + it.each([ + ['https://user:secret@github.com/acme/app.git', 'acme/app'], + ['ssh://git@github.com/acme/app.git', 'acme/app'], + ['git@github.com:acme/app.git', 'acme/app'], + ])('reads GitHub and the repository from %s', (remote, repo) => { + const origin = providerForRemote(remote); + expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual(['github', repo]); }); - it('paginates GitCode open requests before deciding to create a new one', async () => { - vi.spyOn(gitcode, 'getGitCodeToken').mockReturnValue('test-token'); - const pull = { state: 'open', html_url: 'https://gitcode.com/acme/app/pull/2', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }; - const fetcher = vi.fn().mockResolvedValueOnce(new Response(JSON.stringify(Array.from({ length: 100 }, () => ({ ...pull, head: { ...pull.head, ref: 'other' } }))))) - .mockResolvedValueOnce(new Response(JSON.stringify([pull]))); - vi.stubGlobal('fetch', fetcher); - expect(await new GitCodeProvider().findOpenPullRequest(opts)).toBe(pull.html_url); - expect(String(fetcher.mock.calls[1]?.[0])).toContain('page=2'); - }); + it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme'])( + 'reaches %s through git alone, which opens no pull request', + (remote) => expect(providerForRemote(remote)).toEqual({ kind: 'transport' }), + ); - it('uses the CNB CLI schema and normalizes refs/heads branch names', async () => { - const pull = { state: 'open', number: '12', head: { ref: `refs/heads/${opts.source}`, repo: { path: opts.repo } }, base: { ref: 'refs/heads/main', repo: { path: opts.repo } } }; - const cli = vi.spyOn(cnb, 'cnbExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([pull]) }); - const provider = new CNBProvider(); - expect(await provider.findOpenPullRequest(opts)).toBe('https://cnb.cool/acme/app/-/pulls/12'); - expect(cli.mock.calls[0]?.[0]).toContain('list-pulls'); - cli.mockReturnValueOnce({ status: 0, stderr: '', stdout: JSON.stringify(pull) }); - expect(await provider.validatePullRequest('https://cnb.cool/acme/app/-/pulls/12', opts)).toBe(true); + it('a generic git host has no pull request lookup', () => { + expect(new GenericGitProvider().pullRequests).toBeNull(); }); }); diff --git a/src/providers/cnb/index.ts b/src/providers/cnb/index.ts index 2eb62aa2e..903fa52ee 100644 --- a/src/providers/cnb/index.ts +++ b/src/providers/cnb/index.ts @@ -1,5 +1,4 @@ import type { GitProvider, RepoInfo, PrCreateOptions } from '../types.js'; -import { findCnbPull, validateCnbPull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { cnbParseRepoInput, @@ -20,9 +19,9 @@ import { * the TGit provider's "thin class over a platform CLI" shape. */ export class CNBProvider implements GitProvider { - findOpenPullRequest(opts: PrCreateOptions): Promise { return findCnbPull(opts); } - validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateCnbPull(url, opts); } readonly name = 'cnb'; + /** Workflow delivery cannot look up CNB pull requests yet, so it opens none. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return cnbParseRepoInput(input); diff --git a/src/providers/cnb/workflow-pr.ts b/src/providers/cnb/workflow-pr.ts deleted file mode 100644 index 23fde2e45..000000000 --- a/src/providers/cnb/workflow-pr.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { parse } from 'yaml'; -import type { PrCreateOptions } from '../types.js'; -import { CNB_HOST, cnbExec } from './cnb-cli.js'; - -interface Repository { path?: string; web_url?: string } -interface Pull { number: string; state: string; head: { ref: string; repo?: Repository }; base: { ref: string; repo?: Repository } } -function matches(pull: Pull, opts: PrCreateOptions): boolean { - const ref = (value: string): string => value.replace(/^refs\/heads\//, ''); - const repoMatches = (repo: Repository | undefined): boolean => { - if (repo?.web_url) { - const url = new URL(repo.web_url); - return url.host === CNB_HOST && url.pathname.replace(/^\/|\/$/g, '') === opts.repo; - } - return !repo?.path || repo.path === opts.repo || repo.path === opts.repo.split('/').at(-1); - }; - return pull.state === 'open' && ref(pull.head.ref) === opts.source && ref(pull.base.ref) === opts.target - && repoMatches(pull.head.repo) && repoMatches(pull.base.repo); -} -function request(command: string, args: string[], opts: PrCreateOptions): unknown { - const response = cnbExec(['pulls', command, '--repo', opts.repo, ...args], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); - if (response.status !== 0) throw new Error('CNB PR lookup failed. Check cnb authentication or CNB_TOKEN and repository access.'); - try { return parse(response.stdout) as unknown; } - catch { throw new Error('CNB PR lookup returned invalid data. Inspect cnb before retrying.'); } -} -export async function findCnbPull(opts: PrCreateOptions): Promise { - for (let page = 1; page <= 20; page += 1) { - const pulls = request('list-pulls', ['--state', 'open', '--base-ref', opts.target, '--page-size', '100', '--page', String(page)], opts) as Pull[]; - if (!Array.isArray(pulls)) throw new Error('CNB PR lookup returned no request list. Inspect cnb before retrying.'); - const found = pulls.find((pull) => matches(pull, opts)); - if (found) return `https://${CNB_HOST}/${opts.repo}/-/pulls/${found.number}`; - if (pulls.length < 100) return undefined; - } - throw new Error('Too many open CNB requests to establish whether this branch already has one. Inspect it manually.'); -} -export async function validateCnbPull(value: string, opts: PrCreateOptions): Promise { - const url = new URL(value); - if (url.host !== CNB_HOST) return undefined; - const prefix = `/${opts.repo}/-/pulls/`; - if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; - return matches(request('get-pull', ['--number', url.pathname.slice(prefix.length).replace(/\/$/, '')], opts) as Pull, opts); -} diff --git a/src/providers/git/index.ts b/src/providers/git/index.ts index 200b79ced..706809bbe 100644 --- a/src/providers/git/index.ts +++ b/src/providers/git/index.ts @@ -57,6 +57,8 @@ function gitIdentity(): string { */ export class GenericGitProvider implements GitProvider { readonly name = 'git'; + /** Git alone opens no pull request. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return parseGenericGitRepoInput(input); diff --git a/src/providers/gitcode/index.ts b/src/providers/gitcode/index.ts index e710844a8..6370e003e 100644 --- a/src/providers/gitcode/index.ts +++ b/src/providers/gitcode/index.ts @@ -1,5 +1,4 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; -import { findGitCodePull, validateGitCodePull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { gitcodeIsAuthenticated, @@ -26,9 +25,9 @@ import { askQuestion, isInteractive } from '../../utils/prompt.js'; * that distinguish it from the GitLab provider it is structurally modelled on. */ export class GitCodeProvider implements GitProvider { - findOpenPullRequest(opts: PrCreateOptions): Promise { return findGitCodePull(opts); } - validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateGitCodePull(url, opts); } readonly name = 'gitcode'; + /** Workflow delivery cannot look up GitCode pull requests yet, so it opens none. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return parseGitCodeRepoInput(input); diff --git a/src/providers/gitcode/workflow-pr.ts b/src/providers/gitcode/workflow-pr.ts deleted file mode 100644 index 675337d82..000000000 --- a/src/providers/gitcode/workflow-pr.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { PrCreateOptions } from '../types.js'; -import { getGitCodeToken, gitcodeApiBase } from './gitcode-api.js'; -import { GITCODE_HOST } from './repo-url.js'; - -interface Pull { html_url: string; state: string; head: { ref: string; repo?: { full_name?: string } }; base: { ref: string; repo?: { full_name?: string } } } -function matches(pull: Pull, opts: PrCreateOptions): boolean { - return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target - && (!pull.head.repo?.full_name || pull.head.repo.full_name === opts.repo) - && (!pull.base.repo?.full_name || pull.base.repo.full_name === opts.repo); -} -async function request(route: string, opts: PrCreateOptions): Promise { - const token = getGitCodeToken(); - if (!token) throw new Error('GitCode authentication unavailable. Set GITCODE_TOKEN before resuming.'); - const response = await fetch(`${gitcodeApiBase()}/repos/${opts.repo}/pulls${route}`, { - headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' }, redirect: 'manual', - signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), - }); - if (!response.ok) throw new Error(`GitCode PR lookup failed (${response.status}). Check token and repository access.`); - return response.json().catch(() => { throw new Error('gitcode review lookup returned invalid data.'); }); -} -export async function findGitCodePull(opts: PrCreateOptions): Promise { - for (let page = 1; page <= 20; page += 1) { - const query = new URLSearchParams({ state: 'open', base: opts.target, per_page: '100', page: String(page) }); - const pulls = await request(`?${query}`, opts) as Pull[]; - const found = pulls.find((pull) => matches(pull, opts)); - if (found) return found.html_url; - if (pulls.length < 100) return undefined; - } - throw new Error('Too many open GitCode requests to establish whether this branch already has one. Inspect it manually.'); -} -export async function validateGitCodePull(value: string, opts: PrCreateOptions): Promise { - const url = new URL(value); - if (url.hostname !== GITCODE_HOST) return undefined; - const prefix = `/${opts.repo}/pull/`; - if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; - return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as Pull, opts); -} diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index 0f3eb8fe7..43d105ed0 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -1,8 +1,10 @@ import { spawnSync } from 'node:child_process'; import crossSpawn from 'cross-spawn'; +import { z } from 'zod'; import { log } from '../../utils/logger.js'; import { resolveCliPath } from '../../utils/cli-path.js'; import { isInteractive } from '../../utils/prompt.js'; +import type { PullRequestRef } from '../types.js'; // ─── Constants ─────────────────────────────────────────── @@ -594,3 +596,52 @@ async function ghPrCreateViaApi(opts: GhPrCreateOptions): Promise { } return pr.html_url; } + +// ─── Open pull request lookup ──────────────────────────── + +const GhPullSchema = z.object({ + html_url: z.string(), + state: z.string(), + head: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }).nullable() }), + base: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }) }), +}); + +/** + * The open pull request from `source` into `target` in `repo`, asked of an installed gh or, without one, of the API with + * GITHUB_TOKEN or GH_TOKEN. A request from a fork with the same branch name is not it. Never logs in or prompts, and + * throws, rather than answering none, when GitHub cannot be asked or answers what this cannot read. + */ +export async function ghFindOpenPull({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const owner = repo.split('/')[0] ?? repo; + const query = new URLSearchParams({ state: 'open', head: `${owner}:${source}`, base: target, per_page: '100' }); + const pulls = z.array(GhPullSchema).safeParse(await ghApiGet(`repos/${repo}/pulls?${query}`, timeoutMs)); + if (!pulls.success) throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + const ours = (name: string | undefined): boolean => name?.toLowerCase() === repo.toLowerCase(); + return pulls.data.find((pull) => pull.state === 'open' && pull.head.ref === source && pull.base.ref === target + && ours(pull.head.repo?.full_name) && ours(pull.base.repo.full_name))?.html_url; +} + +async function ghApiGet(path: string, timeoutMs: number): Promise { + if (isGhInstalled()) { + const result = ghExec(['api', path], { timeoutMs, env: { GH_PROMPT_DISABLED: '1', GIT_TERMINAL_PROMPT: '0' } }); + if (result.status !== 0) throw new Error('gh could not look up open pull requests. Check gh auth status and its access to the repository.'); + try { + return JSON.parse(result.stdout); + } catch { + throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + } + } + const token = getGitHubToken(); + if (!token) throw new Error('GitHub credentials are unavailable: install gh and log in, or set GITHUB_TOKEN or GH_TOKEN.'); + const response = await fetch(`${GITHUB_API}/${path}`, { + headers: { 'Accept': 'application/vnd.github+json', 'Authorization': `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, + redirect: 'manual', + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`GitHub answered the open pull request lookup with HTTP ${response.status}. Check the token's access to the repository.`); + try { + return await response.json(); + } catch { + throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + } +} diff --git a/src/providers/github/index.ts b/src/providers/github/index.ts index b24c3fbf7..4ef461626 100644 --- a/src/providers/github/index.ts +++ b/src/providers/github/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGhAvailable, @@ -8,14 +8,15 @@ import { ghRepoClone, ghCreateRepo, ghPrCreate, + ghFindOpenPull, RepoNotFoundError as GhRepoNotFoundError, } from './gh-cli.js'; import { ghListOrgRepos } from './gh-org.js'; import { parseGitHubRepoInput } from './repo-url.js'; -import { findGitHubPull, validateGitHubPull } from './workflow-pr.js'; export class GitHubProvider implements GitProvider { readonly name = 'github'; + readonly pullRequests: PullRequestLookup = { findOpen: ghFindOpenPull }; parseRepoInput(input: string): RepoInfo { return parseGitHubRepoInput(input); @@ -65,14 +66,6 @@ export class GitHubProvider implements GitProvider { }); } - findOpenPullRequest(opts: PrCreateOptions): Promise { - return findGitHubPull(opts); - } - - validatePullRequest(url: string, opts: PrCreateOptions): Promise { - return validateGitHubPull(url, opts); - } - getDefaultEmailDomain(): string | null { return null; } diff --git a/src/providers/github/workflow-pr.ts b/src/providers/github/workflow-pr.ts deleted file mode 100644 index dde389cd4..000000000 --- a/src/providers/github/workflow-pr.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { PrCreateOptions } from '../types.js'; -import { getGitHubToken, ghExec, isGhInstalled } from './gh-cli.js'; - -interface Pull { - html_url: string; - state: string; - head: { ref: string; repo: { full_name: string } | null }; - base: { ref: string; repo: { full_name: string } }; -} - -async function request(path: string, opts: PrCreateOptions): Promise { - if (isGhInstalled()) { - const result = ghExec(['api', path], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); - if (result.status !== 0) throw new Error('GitHub PR lookup failed. Check gh authentication and repository access.'); - try { return JSON.parse(result.stdout) as unknown; } - catch { throw new Error('GitHub PR lookup returned invalid data. Inspect gh before retrying.'); } - } - const token = getGitHubToken(); - if (!token) throw new Error('GitHub authentication unavailable. Set GITHUB_TOKEN or authenticate gh before resuming.'); - const response = await fetch(`https://api.github.com/${path}`, { - headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, - redirect: 'manual', - signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), - }); - if (!response.ok) throw new Error(`GitHub PR lookup failed (${response.status}). Check token and repository access.`); - return response.json().catch(() => { throw new Error('GitHub PR lookup returned invalid data.'); }); -} - -function matches(pull: Pull, opts: PrCreateOptions): boolean { - return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target - && pull.head.repo?.full_name.toLowerCase() === opts.repo.toLowerCase() - && pull.base.repo.full_name.toLowerCase() === opts.repo.toLowerCase(); -} - -export async function findGitHubPull(opts: PrCreateOptions): Promise { - const query = new URLSearchParams({ state: 'open', head: `${opts.repo.split('/')[0]}:${opts.source}`, base: opts.target, per_page: '100' }); - const pulls = await request(`repos/${opts.repo}/pulls?${query}`, opts) as Pull[]; - return pulls.find((pull) => matches(pull, opts))?.html_url; -} - -export async function validateGitHubPull(value: string, opts: PrCreateOptions): Promise { - const url = new URL(value); - // A remote driver may use a separate review service. Validate recognized provider URLs only. - if (url.hostname !== 'github.com') return undefined; - const match = /^\/([^/]+\/[^/]+)\/pull\/(\d+)\/?$/.exec(url.pathname); - if (!match || match[1]!.toLowerCase() !== opts.repo.toLowerCase()) return false; - return matches(await request(`repos/${opts.repo}/pulls/${match[2]}`, opts) as Pull, opts); -} diff --git a/src/providers/gitlab/index.ts b/src/providers/gitlab/index.ts index 5d05e8f90..5a603cee5 100644 --- a/src/providers/gitlab/index.ts +++ b/src/providers/gitlab/index.ts @@ -13,7 +13,6 @@ import { gitlabListOrgRepos } from './org.js'; import { fetchGitLabMR } from './mr-fetch.js'; import { parseGitLabRepoInput } from './repo-url.js'; import type { MRData } from '../../types.js'; -import { findGitLabMergeRequest, validateGitLabMergeRequest } from './workflow-pr.js'; /** * GitLab provider (supports self-hosted / enterprise GitLab instances). @@ -23,6 +22,8 @@ import { findGitLabMergeRequest, validateGitLabMergeRequest } from './workflow-p */ export class GitLabProvider implements GitProvider { readonly name = 'gitlab'; + /** Workflow delivery cannot look up GitLab merge requests yet, so it opens none. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return parseGitLabRepoInput(input); @@ -81,14 +82,6 @@ export class GitLabProvider implements GitProvider { return fetchGitLabMR(url); } - findOpenPullRequest(opts: PrCreateOptions): Promise { - return findGitLabMergeRequest(opts); - } - - validatePullRequest(url: string, opts: PrCreateOptions): Promise { - return validateGitLabMergeRequest(url, opts); - } - async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gitlabListOrgRepos(org, opts); } diff --git a/src/providers/gitlab/workflow-pr.ts b/src/providers/gitlab/workflow-pr.ts deleted file mode 100644 index cf8c1ed9a..000000000 --- a/src/providers/gitlab/workflow-pr.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { PrCreateOptions } from '../types.js'; -import { getGitLabToken, gitlabApiBase, gitlabBaseUrl } from './gitlab-api.js'; - -interface MergeRequest { - web_url: string; - state: string; - source_branch: string; - target_branch: string; - source_project_id: number; - target_project_id: number; -} - -function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { - return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target - && mr.source_project_id === mr.target_project_id; -} - -async function request(route: string, opts: PrCreateOptions): Promise { - const token = getGitLabToken(); - if (!token) throw new Error('GitLab authentication unavailable. Set GITLAB_TOKEN before resuming.'); - const response = await fetch(`${gitlabApiBase()}/projects/${encodeURIComponent(opts.repo)}/merge_requests${route}`, { - headers: { 'PRIVATE-TOKEN': token, Accept: 'application/json' }, redirect: 'manual', - signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), - }); - if (!response.ok) throw new Error(`GitLab MR lookup failed (${response.status}). Check token and repository access.`); - return response.json().catch(() => { throw new Error('gitlab review lookup returned invalid data.'); }); -} - -export async function findGitLabMergeRequest(opts: PrCreateOptions): Promise { - const query = new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100' }); - const requests = await request(`?${query}`, opts) as MergeRequest[]; - return requests.find((mr) => matches(mr, opts))?.web_url; -} - -export async function validateGitLabMergeRequest(value: string, opts: PrCreateOptions): Promise { - const base = new URL(gitlabBaseUrl()); - const url = new URL(value); - if (url.origin !== base.origin) return undefined; - const prefix = `${base.pathname.replace(/\/$/, '')}/${opts.repo}/-/merge_requests/`; - if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; - return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as MergeRequest, opts); -} diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 36057530f..5d369373c 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1,4 +1,4 @@ -import type { GitProvider } from './types.js'; +import type { GitProvider, PullRequestLookup } from './types.js'; import { TGitProvider } from './tgit/index.js'; import { GitHubProvider } from './github/index.js'; import { CNBProvider } from './cnb/index.js'; @@ -213,3 +213,50 @@ export function getProviderFromUrl(repoUrl: string): GitProvider { const name = detectProvider(repoUrl); return getProvider(name); } + +// ─── Workflow delivery ─────────────────────────────────── + +/** A checkout's origin as workflow delivery reaches it: through a provider that can look up its pull requests, or git alone. */ +export type OriginProvider = + | { kind: 'provider'; provider: GitProvider; pullRequests: PullRequestLookup; repo: string; host: string } + | { kind: 'transport' }; + +/** + * The provider of a checkout's configured origin URL, `remote.origin.url` as written before `insteadOf` rewrites it, and + * the repository on it. Credentials, query and fragment are dropped first. A filesystem path, a file:// URL, a host no + * provider owns, a provider with no pull request lookup, or a URL its provider cannot parse is reached through git alone. + * The bare `owner/repo` default is for user input, never for a remote. + */ +export function providerForRemote(remoteUrl: string): OriginProvider { + const hosted = hostingUrl(remoteUrl.trim()); + if (!hosted) return { kind: 'transport' }; + const provider = getProviderFromUrl(hosted.href); + if (!provider.pullRequests) return { kind: 'transport' }; + try { + const { owner, repo } = provider.parseRepoInput(hosted.href); + return { kind: 'provider', provider, pullRequests: provider.pullRequests, repo: `${owner}/${repo}`, host: hosted.host }; + } catch { + return { kind: 'transport' }; + } +} + +/** The web URL of a remote in its https, ssh or scp form, without credentials, query or fragment; undefined for any other form. */ +function hostingUrl(remote: string): URL | undefined { + const scp = /^[^@\s/]+@([^:\s/]+):(.+)$/.exec(remote); + if (scp) { + const [, host, repoPath] = scp; + return host && repoPath ? parsedUrl(`https://${host}/${repoPath}`) : undefined; + } + if (!/^(https?|ssh):\/\//i.test(remote)) return undefined; + const url = parsedUrl(remote); + if (!url) return undefined; + return parsedUrl(url.protocol === 'ssh:' ? `https://${url.hostname}${url.pathname}` : `${url.protocol}//${url.host}${url.pathname}`); +} + +function parsedUrl(value: string): URL | undefined { + try { + return new URL(value); + } catch { + return undefined; + } +} diff --git a/src/providers/tgit/index.ts b/src/providers/tgit/index.ts index ddc583617..8ae8c58c4 100644 --- a/src/providers/tgit/index.ts +++ b/src/providers/tgit/index.ts @@ -1,5 +1,4 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; -import { findTGitMergeRequest, validateTGitMergeRequest } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGfInstalled, @@ -16,6 +15,8 @@ import { parseTGitRepoInput } from './repo-url.js'; export class TGitProvider implements GitProvider { readonly name = 'tgit'; + /** Workflow delivery cannot look up TGit merge requests yet, so it opens none. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return parseTGitRepoInput(input); @@ -69,9 +70,6 @@ export class TGitProvider implements GitProvider { return 'tencent.com'; } - findOpenPullRequest(opts: PrCreateOptions): Promise { return findTGitMergeRequest(opts); } - validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateTGitMergeRequest(url, opts); } - async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gfListOrgRepos(org, opts); } diff --git a/src/providers/tgit/workflow-pr.ts b/src/providers/tgit/workflow-pr.ts deleted file mode 100644 index 96b7c7c36..000000000 --- a/src/providers/tgit/workflow-pr.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { PrCreateOptions } from '../types.js'; -import { tgitFetch } from './rest-auth.js'; - -interface MergeRequest { web_url: string; state: string; source_branch: string; target_branch: string; source_project_id: number; target_project_id: number } -function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { - return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target - && mr.source_project_id === mr.target_project_id; -} -async function list(opts: PrCreateOptions, query: URLSearchParams): Promise { - const response = await tgitFetch(`/projects/${encodeURIComponent(opts.repo)}/merge_requests?${query}`, { - redirect: 'manual', signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), - }); - if (!response.ok) throw new Error(`TGit MR lookup failed (${response.status}). Check TGIT_TOKEN or gf authentication.`); - return response.json().catch(() => { throw new Error('TGit MR lookup returned invalid data.'); }) as Promise; -} -export async function findTGitMergeRequest(opts: PrCreateOptions): Promise { - for (let page = 1; page <= 20; page += 1) { - const requests = await list(opts, new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100', page: String(page) })); - const found = requests.find((mr) => matches(mr, opts)); - if (found) return found.web_url; - if (requests.length < 100) return undefined; - } - throw new Error('Too many open TGit requests to establish whether this branch already has one. Inspect it manually.'); -} -export async function validateTGitMergeRequest(value: string, opts: PrCreateOptions): Promise { - const url = new URL(value); - if (url.hostname !== 'git.woa.com') return undefined; - const match = /^\/(.+?)\/(?:-\/)?merge_requests\/(\d+)\/?$/.exec(url.pathname); - if (!match || match[1] !== opts.repo) return false; - const requests = await list(opts, new URLSearchParams({ iid: match[2]! })); - return requests.length === 1 && matches(requests[0]!, opts); -} diff --git a/src/providers/types.ts b/src/providers/types.ts index 4b2bae05d..166569c95 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -48,6 +48,25 @@ export interface PrCreateOptions { spawnTimeoutMs?: number; } +/** The pull request workflow delivery looks for: head branch `source` into `target`, both in `repo`. */ +export interface PullRequestRef { + /** Repository in "owner/repo" format, as parseRepoInput gives it. */ + repo: string; + source: string; + target: string; + /** How long the host may take to answer. */ + timeoutMs: number; +} + +/** + * Read-only pull request lookup for workflow delivery. It has no merge, close, approve or update member, and a test + * pins its key set. It never prompts, logs in or calls authenticate(). + */ +export interface PullRequestLookup { + /** The web URL of the open request `ref` names, if any. Throws when the host cannot answer, never answering "none". */ + findOpen(ref: PullRequestRef): Promise; +} + /** * 轻量级仓库元信息，用于 listOrgRepos 返回。 */ @@ -120,11 +139,11 @@ export interface GitProvider { */ createPullRequest(opts: PrCreateOptions): Promise; - /** Finds an already open request, so a delivery restarted after a crash does not create another one. */ - findOpenPullRequest?(opts: PrCreateOptions): Promise; - - /** False means known metadata disagrees; undefined means this provider cannot validate the URL. */ - validatePullRequest?(url: string, opts: PrCreateOptions): Promise; + /** + * How workflow delivery finds the open request of a branch before it opens one, so a delivery that ran before opens + * no second request. Null for a host only git reaches, where delivery opens none. + */ + readonly pullRequests: PullRequestLookup | null; /** * 获取指定 MR/PR 的完整数据（标题、描述、提交列表、diff）。 diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index b6a21e115..c601a9a8a 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -906,7 +906,7 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { continue; } return { - key, runId: run.id, checkout: run.checkout, base: run.base.commit, target: run.base.ref, + key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target: run.base.ref, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } @@ -931,6 +931,23 @@ function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutc return { requeueEnd: receipted && run.status !== 'running' }; } +/** + * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still + * delivers `job`. If it does, it is marked as creating in the same write, so a request opened by a process that then + * stops is looked up rather than opened again. + */ +async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob): Promise { + const marked = await updateRun(config, runId, async (run) => { + if (run === 'missing' || run.status !== 'running') return { value: false }; + const settledRun = reconcileRun(run); + const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); + if (!step) return { value: false }; + step.delivery.creating = true; + return { value: true, write: { ...run, updatedAt: new Date().toISOString() } }; + }); + return marked.status === 'done' && marked.value; +} + /** * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it. The run lock is * held to pick a delivery and to apply its outcome, never across git or a provider. While another process of this @@ -951,7 +968,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise markCreating(config, runId, job)); const applied = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const { requeueEnd } = completeDelivery(run, job, outcome); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index cc81f7df1..0ee6edfd3 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -3,7 +3,10 @@ * engine builds a job under the run lock, calls `deliverPr` once it released the lock, and applies the outcome under * the lock again, so no network call holds the lock and this module needs nothing of the engine's. */ -import type { PrUrl } from './pr-url.js'; +import os from 'node:os'; +import { providerForRemote } from '../providers/registry.js'; +import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; +import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import type { WorkflowFailure } from './types.js'; import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; @@ -20,6 +23,7 @@ export interface DeliveryKey { export interface DeliveryJob { key: DeliveryKey; runId: string; + runName: string; /** The checkout the run started or was claimed in, which reads the delivered branch. */ checkout: string; /** The run base commit the delivered branch is compared with. */ @@ -35,24 +39,79 @@ export type DeliveryFault = | { kind: 'no_checkout' } | { kind: 'unreadable' } | { kind: 'protected_path'; files: string[] } - | { kind: 'no_provider' }; + | { kind: 'no_provider' } + | { kind: 'lookup_failed'; provider: string } + | { kind: 'create_failed'; provider: string } + | { kind: 'unsafe_url'; provider: string } + | { kind: 'withdrawn' }; export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; +/** + * Asked under the run lock right before a pull request is opened: true while the step still delivers the job, which it + * then marks as creating. False opens nothing. + */ +export type CreateGate = () => Promise; + +/** How long a host may take to answer one lookup or one create. */ +export const DELIVERY_TIMEOUT_MS = 30_000; + /** A full object id: the head comes from a driver's report, and anything else could reach git as an option. */ const OBJECT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/i; +const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }); + /** * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, - * so no pull request goes out, and no step succeeds, with a protected path changed. + * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch + * is looked up, and only when there is none, and `gate` agrees, is one opened; a create that fails is looked up again. + * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ -export async function deliverPr(job: DeliveryJob): Promise { +export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { const changed = await changedFiles(job); - if ('fault' in changed) return { ok: false, fault: changed.fault }; + if ('fault' in changed) return failed(changed.fault); const files = protectedPaths(changed.files); - if (files.length > 0) return { ok: false, fault: { kind: 'protected_path', files } }; + if (files.length > 0) return failed({ kind: 'protected_path', files }); if (job.reported) return { ok: true, pr: job.reported }; - return { ok: false, fault: { kind: 'no_provider' } }; + const remote = await git(job.checkout, ['config', '--get', 'remote.origin.url']); + const origin = remote.code === 0 ? providerForRemote(remote.stdout) : undefined; + if (!origin || origin.kind === 'transport') return failed({ kind: 'no_provider' }); + const { provider, pullRequests, repo } = origin; + const ref: PullRequestRef = { repo, source: job.key.branch, target: job.target, timeoutMs: DELIVERY_TIMEOUT_MS }; + const found = await findOpen(pullRequests, ref); + if (found.kind === 'found') return { ok: true, pr: found.pr }; + if (found.kind === 'failed') return failed({ kind: 'lookup_failed', provider: provider.name }); + if (found.kind === 'unsafe') return failed({ kind: 'unsafe_url', provider: provider.name }); + if (!await gate()) return failed({ kind: 'withdrawn' }); + try { + const created = sanitizePrUrl(await provider.createPullRequest({ + repo, source: job.key.branch, target: job.target, title: `${job.runName}: ${job.key.step}`, + description: `Opened by TeamAI for review: workflow ${job.runName}, run ${job.runId}, step ${job.key.step}, pass ${job.key.pass}.\n\nBase commit: ${job.base}\n\nTeamAI never merges this pull request.`, + // Run outside the member checkout, so the provider's CLI can neither push from it nor read its branch. + cwd: os.tmpdir(), + spawnTimeoutMs: DELIVERY_TIMEOUT_MS, + })); + return created.url ? { ok: true, pr: created.url } : failed({ kind: 'unsafe_url', provider: provider.name }); + } catch { + // The host may have opened the request before the create failed or timed out: find it rather than open another. + const again = await findOpen(pullRequests, ref); + return again.kind === 'found' ? { ok: true, pr: again.pr } : failed({ kind: 'create_failed', provider: provider.name }); + } +} + +/** What a lookup said: the open request, sanitized; none; no answer; or a URL that cannot be kept. */ +async function findOpen( + lookup: PullRequestLookup, ref: PullRequestRef, +): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' } | { kind: 'unsafe' }> { + let url: string | undefined; + try { + url = await lookup.findOpen(ref); + } catch { + return { kind: 'failed' }; + } + if (url === undefined) return { kind: 'none' }; + const clean = sanitizePrUrl(url); + return clean.url ? { kind: 'found', pr: clean.url } : { kind: 'unsafe' }; } /** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ @@ -95,6 +154,26 @@ export function deliveryFailure({ runId, key }: Pick; /** * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. + * `creating` is set under the run lock right before TeamAI asks the host to open a request, and stays: a request may + * exist with no receipt after a crash or a cancel, so such a step is looked up before anything opens another. */ -const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); +const DeliverySchema = z.object({ + branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional(), creating: z.literal(true).optional(), +}); export type Delivery = z.infer; /** @@ -166,7 +170,10 @@ const SucceededSchema = StepBaseSchema.extend({ }); const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional() }); const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); -const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema }); +const CancelledSchema = StepBaseSchema.extend({ + status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema, + delivery: DeliverySchema.optional(), +}); const LeasedSchema = StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }); /** @@ -315,10 +322,16 @@ export function fail(state: StepState, settlement: Settlement): Fail return become(state, { status: 'failed', outcome: 'failed', ...settlement }); } -/** An open step is cancelled before it settles. A leased or delivering one keeps the attempt it no longer settles. */ +/** + * An open step is cancelled before it settles. A leased or delivering one keeps the attempt it no longer settles, and a + * delivering one what it was delivering, since a request it asked for may exist with no receipt yet. + */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { const attempt = state.status === 'leased' || state.status === 'delivering' ? state.attempt : undefined; - return become(state, { status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), failure }); + const delivery = state.status === 'delivering' ? state.delivery : undefined; + return become(state, { + status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), ...(delivery ? { delivery } : {}), failure, + }); } /** The current pass of a settled step, as `passes` keeps it once a loop runs the step again; undefined while it has not settled. */ From 5db2f040d201862f38b0988aee19a898a1677d47 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:26:46 +0200 Subject: [PATCH 11/77] fix(workflow): keep a reported pull request on the origin host only when it is the branch's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A driver could report any sanitized URL on the origin's host for a step with deliver: pr. Delivery then recorded it as the step's pull request, even when it named another branch's request or no open request at all. Before the run lock is taken, the report now asks the origin's provider for the open request of the reported branch into the run base branch. A URL on that host is kept only when it equals that request. Otherwise it is dropped with a reason that names the branch, and delivery finds or opens the branch's request itself, so a dropped alternate route of the same request costs nothing. A URL on another host, an origin only git reaches, or a lookup that cannot answer keeps the URL as sanitized. No per-provider URL parsing is involved. The protected-path test with a reported URL now seeds its open request, so the URL is actually kept. It asserts that nothing is opened or accepted, rather than that no provider is asked, since the report-time lookup only reads. Tests that report a github.com URL against the fixture origin now use the fake gh, so none of them reaches GitHub. Red: both "drops a URL on the origin host that ..." cases failed with `expected { data: { accepted: true, …(3) } } to match object { data: { accepted: true, …(2) } }`, since the URL was kept and nothing was dropped. --- src/__tests__/workflow-delivery.test.ts | 45 ++++++++++++++++++++++--- src/providers/registry.ts | 4 +-- src/workflow/engine.ts | 22 +++++++++--- src/workflow/pr-delivery.ts | 29 +++++++++++++--- 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 87539b233..2fd2a3fe5 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -340,10 +340,10 @@ describe('delivering a step', () => { it.each([ ['TeamAI would open the pull request', {}], ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], - ])('refuses a branch that changes protected paths, naming them, before any provider request, when %s', async (_case, changes) => { + ])('refuses a branch that changes protected paths, naming them, before any pull request is opened or accepted, when %s', async (_case, changes) => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); - const gh = fakeGh(path.join(root, 'gh')); const fix = await take(config, runId); + const gh = fakeGh(path.join(root, 'gh'), { open: [pull(1, `teamai/${runId}/fix/p1`)] }); const to = pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n', 'src/app.ts': 'ok\n', 'packages/web/CLAUDE.md': 'rules\n' }); await succeed(config, runId, fix, { to, ...changes }); @@ -354,12 +354,15 @@ describe('delivering a step', () => { { id: 'review', status: 'skipped' }, ]); expect(run.steps[0]).not.toMatchObject({ failure: { message: expect.stringContaining('src/app.ts') } }); + if ('pr_url' in changes) expect(run.steps[0]).toMatchObject({ delivery: { reported: changes.pr_url } }); expect(run.deliveries).toBeUndefined(); - expect(gh.calls()).toEqual([]); + // The report-time lookup only reads; delivery refuses the protected paths before it looks up or opens anything. + expect(gh.calls().map((call) => call.args[0])).toEqual('pr_url' in changes ? ['api'] : []); }); it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + fakeGh(path.join(root, 'gh')); const fix = await take(config, runId); pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n' }); await succeed(config, runId, fix, { to: 'main', pr_url: 'https://github.com/fixture/app/pull/1' }); @@ -458,11 +461,45 @@ describe('reported URLs from a driver', () => { }); it('drops a URL from a step that does not deliver, saying why, and still accepts the report', async () => { - const { config, runId } = await fixture(' - id: write\n command: node write.js\n workspace: new\n'); + const { config, root, runId } = await fixture(' - id: write\n command: node write.js\n workspace: new\n'); + const gh = fakeGh(path.join(root, 'gh')); expect(await succeed(config, runId, await take(config, runId), { pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ data: { accepted: true, revision: 2, step: { id: 'write', status: 'succeeded', outcome: 'success' }, cleanup: true, dropped: [{ field: 'changes.pr_url', reason: 'the step has no deliver: pr' }], } }); expect((await state(config, runId)).deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + + it.each([ + ['is not the open pull request of its branch', [7], (branch: string) => `is not the open pull request of branch ${branch} into main`, 7], + ['names a branch with no open pull request', [], (branch: string) => `branch ${branch} has no open pull request into main`, 1], + ])('drops a URL on the origin host that %s, and delivers the open one of the branch instead', async (_case, open, reason, delivered) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + fakeGh(path.join(root, 'gh'), { open: open.map((number) => pull(number, branch)) }); + expect(await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/9' })).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: reason(branch) }], + } }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual([`https://github.com/fixture/app/pull/${delivered}`]); + }); + + it.each([ + ['is the open pull request of its branch', {}], + ['cannot be checked while GitHub cannot be reached', { lookup: 'fail' as const }], + ])('keeps a URL on the origin host that %s, and opens nothing for it', async (_case, options) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const gh = fakeGh(path.join(root, 'gh'), { ...options, open: [pull(4, `teamai/${runId}/fix/p1`)] }); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/4?tab=files#top' }); + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'fix', status: 'delivering' } } }); + expect(accepted).not.toHaveProperty('data.dropped'); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/4']); + expect(gh.calls().filter((call) => call.args[0] === 'pr')).toEqual([]); }); }); diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 5d369373c..f2a20c7cb 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -218,7 +218,7 @@ export function getProviderFromUrl(repoUrl: string): GitProvider { /** A checkout's origin as workflow delivery reaches it: through a provider that can look up its pull requests, or git alone. */ export type OriginProvider = - | { kind: 'provider'; provider: GitProvider; pullRequests: PullRequestLookup; repo: string; host: string } + | { kind: 'provider'; provider: GitProvider; pullRequests: PullRequestLookup; repo: string; hostname: string } | { kind: 'transport' }; /** @@ -234,7 +234,7 @@ export function providerForRemote(remoteUrl: string): OriginProvider { if (!provider.pullRequests) return { kind: 'transport' }; try { const { owner, repo } = provider.parseRepoInput(hosted.href); - return { kind: 'provider', provider, pullRequests: provider.pullRequests, repo: `${owner}/${repo}`, host: hosted.host }; + return { kind: 'provider', provider, pullRequests: provider.pullRequests, repo: `${owner}/${repo}`, hostname: hosted.hostname }; } catch { return { kind: 'transport' }; } diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index c601a9a8a..b816862db 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -25,8 +25,8 @@ import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, item import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; import { deliverableOf, deliveryProblem, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; -import { deliverPr, deliveryFailure, type DeliveryJob, type DeliveryKey, type DeliveryOutcome } from './pr-delivery.js'; -import { sanitizePrUrl } from './pr-url.js'; +import { deliverPr, deliveryFailure, reportedPrMismatch, type DeliveryJob, type DeliveryKey, type DeliveryOutcome } from './pr-delivery.js'; +import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; @@ -1404,12 +1404,26 @@ function continueSession(run: RunState, report: ReportInput): void { (run.sessionContinuations ??= []).push({ from, to }); } +/** + * Why the URL a delivering step's report keeps after sanitizing is still dropped, read from the run before its lock is + * taken, since the provider's lookup is a network call. The locked report checks the step again. + */ +async function reportedPrUnmatched(config: LocalConfig, runId: string, report: ReportInput, url: PrUrl): Promise { + if (report.outcome !== 'success' || !report.changes || !isRunId(runId)) return undefined; + const run = await readRun(config, runId); + if (typeof run === 'string' || !run.checkout || !run.base?.ref) return undefined; + const step = run.steps.find((item) => item.status === 'leased' && item.lease.attempt === report.attempt); + if (!step || locateStep(run, step.id)?.step.deliver !== 'pr') return undefined; + return reportedPrMismatch({ checkout: run.checkout, url, branch: report.changes.branch, target: run.base.ref }); +} + export async function reportAttempt(config: LocalConfig, runId: string, report: ReportInput): Promise { if (isLeaseRef(report.attempt)) { return error('judge_lease', `Attempt ${report.attempt} is a judge lease, which workflow report does not settle. Submit its answer with: teamai score --lease ${report.attempt} --submit --stdin`); } const install = await installOwner(config); const reported = report.prUrl === undefined ? undefined : sanitizePrUrl(report.prUrl); + const unmatched = reported?.url ? await reportedPrUnmatched(config, runId, report, reported.url) : undefined; return withRun(config, runId, async (run) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { @@ -1488,13 +1502,13 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: const delivery = outcome === 'success' && defined?.deliver === 'pr' ? deliverableOf(run, step, defined, report.changes) : undefined; const settledStep = outcome === 'released' ? release(step) : outcome !== 'success' ? fail(step, { ...settlement, failure }) - : delivery ? startDelivery(step, successful, { ...delivery, ...(reported?.url ? { reported: reported.url } : {}) }) + : delivery ? startDelivery(step, successful, { ...delivery, ...(reported?.url && !unmatched ? { reported: reported.url } : {}) }) : succeed(step, successful); // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. Delivery reads the // pushed branch, so it does not keep the worktree. const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); // A URL that is not kept is named without its value, which may hold a token, and the report still counts. - const dropReason = reported?.reason ?? (reported && defined?.deliver !== 'pr' ? 'the step has no deliver: pr' : undefined); + const dropReason = reported?.reason ?? unmatched ?? (reported && defined?.deliver !== 'pr' ? 'the step has no deliver: pr' : undefined); return { save: true, envelope: ok({ diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 0ee6edfd3..bb340a162 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -4,7 +4,7 @@ * the lock again, so no network call holds the lock and this module needs nothing of the engine's. */ import os from 'node:os'; -import { providerForRemote } from '../providers/registry.js'; +import { providerForRemote, type OriginProvider } from '../providers/registry.js'; import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import type { WorkflowFailure } from './types.js'; @@ -73,9 +73,8 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise 0) return failed({ kind: 'protected_path', files }); if (job.reported) return { ok: true, pr: job.reported }; - const remote = await git(job.checkout, ['config', '--get', 'remote.origin.url']); - const origin = remote.code === 0 ? providerForRemote(remote.stdout) : undefined; - if (!origin || origin.kind === 'transport') return failed({ kind: 'no_provider' }); + const origin = await originOf(job.checkout); + if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); const { provider, pullRequests, repo } = origin; const ref: PullRequestRef = { repo, source: job.key.branch, target: job.target, timeoutMs: DELIVERY_TIMEOUT_MS }; const found = await findOpen(pullRequests, ref); @@ -99,6 +98,28 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { + const origin = await originOf(checkout); + if (origin.kind === 'transport' || new URL(url).hostname !== origin.hostname) return undefined; + const found = await findOpen(origin.pullRequests, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); + if (found.kind === 'none') return `branch ${branch} has no open pull request into ${target}`; + if (found.kind === 'found' && found.pr !== url) return `is not the open pull request of branch ${branch} into ${target}`; + return undefined; +} + +/** The provider of the checkout's configured origin; git alone when the checkout has none. */ +async function originOf(checkout: string): Promise { + const remote = await git(checkout, ['config', '--get', 'remote.origin.url']); + return remote.code === 0 ? providerForRemote(remote.stdout) : { kind: 'transport' }; +} + /** What a lookup said: the open request, sanitized; none; no answer; or a URL that cannot be kept. */ async function findOpen( lookup: PullRequestLookup, ref: PullRequestRef, From 98ebc9eb121218ae47c7c7beedffd3be714851e3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:45:33 +0200 Subject: [PATCH 12/77] fix(workflow): hand over the branch a copy started from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A copy records the run branch it started from when it is first leased, so a pull request from it can later target the nearest delivered branch it grew from. The field was published by the snapshot rules but the snapshot format had no place for it, so any run where one step's copy starts from another step's branch could not be published or claimed. The field is now `from`, the name the delivery target walk reads, with a place in the snapshot. A claim refuses a snapshot whose copy names a `from` outside the run, as it already refuses a copy branch outside the run, since a pull request may target that branch. Red: "hands over a copy that started from the branch of a step it needs, with the branch it started from" failed with snapshot_unsupported on copies.teamai//b/p1.origin; "refuses to claim a snapshot whose copy started from a branch outside the run" failed with the claim accepted (`expected { ok: true, … } to match object { ok: false, … }`). --- src/__tests__/workflow-handoff.test.ts | 43 ++++++++++++++++++++++---- src/workflow/engine.ts | 11 ++++--- src/workflow/snapshot.ts | 10 ++++-- src/workflow/types.ts | 5 +-- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/__tests__/workflow-handoff.test.ts b/src/__tests__/workflow-handoff.test.ts index f235e73d9..d5cdf8e41 100644 --- a/src/__tests__/workflow-handoff.test.ts +++ b/src/__tests__/workflow-handoff.test.ts @@ -92,6 +92,19 @@ steps: command: echo change > change.txt `; +const STACK = `schemaVersion: v1alpha1 +name: stack +description: A change, and one that starts from it. +steps: + - id: a + workspace: new + command: echo a > a.txt + - id: b + needs: [a] + workspace: new + command: echo b > b.txt +`; + const WORKSPACES = ['read-only', 'new', 'continue'] as const; interface Member { @@ -518,7 +531,7 @@ describe('workflow handoff through teamai-reports', () => { }); interface HandedStep { id: string; changes: { branch: string; base: string; from: string; to: string; commits: string[] }; read?: unknown; passes?: unknown[] } - interface HandedRun { base: { commit: string }; copies: Record; steps: HandedStep[]; archive: unknown[] } + interface HandedRun { base: { commit: string }; copies: Record; steps: HandedStep[]; archive: unknown[] } it.each<[string, (snapshot: HandedRun, step: HandedStep, branch: string) => void, (at: number, branch: string) => string]>([ ['its base', (snapshot) => { snapshot.base.commit = 'main'; }, () => 'base.commit'], ['a copy', (snapshot, _step, branch) => { snapshot.copies[branch] = { commit: 'HEAD' }; }, (_at, branch) => `copies.${branch}.commit`], @@ -560,9 +573,13 @@ describe('workflow handoff through teamai-reports', () => { expect(await readRun(bob.config, runId)).toBe('missing'); }); - it.each([['a branch outside any run', 'main'], ['a branch of another run', 'teamai/r0123456789abcdef/change/p1']])( - 'refuses to claim a snapshot whose copy is %s, and changes nothing', - async (_name, branch) => { + it.each<[string, (branch: string, commit: string) => HandedRun['copies'], (branch: string) => string]>([ + ['is a branch outside any run', (_branch, commit) => ({ main: { commit } }), () => 'copies.main'], + ['is a branch of another run', (_branch, commit) => ({ 'teamai/r0123456789abcdef/change/p1': { commit } }), () => 'copies.teamai/r0123456789abcdef/change/p1'], + ['started from a branch outside the run', (branch, commit) => ({ [branch]: { commit, from: 'main' } }), (branch) => `copies.${branch}.from`], + ])( + 'refuses to claim a snapshot whose copy %s, and changes nothing', + async (_name, copies, field) => { const acme = team({ ship: SHIP }); const alice = await acme.member('alice'); const bob = await acme.member('bob'); @@ -573,12 +590,12 @@ describe('workflow handoff through teamai-reports', () => { git(path.dirname(acme.origin), 'clone', '-q', '--branch', 'teamai-reports', acme.origin, editor); const rel = `workflows/runs/${runId}/state.json`; const snapshot: HandedRun = JSON.parse(fs.readFileSync(path.join(editor, rel), 'utf8')); - snapshot.copies = { [branch]: { commit: wrote.commit } }; + snapshot.copies = copies(wrote.branch, wrote.commit); fs.writeFileSync(path.join(editor, rel), `${JSON.stringify(snapshot, null, 2)}\n`); git(editor, 'commit', '-qam', 'hand edit'); git(editor, 'push', '-q', 'origin', 'teamai-reports'); - expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field: `copies.${branch}` } }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field: field(wrote.branch) } }); expect(await readRun(bob.config, runId)).toBe('missing'); }, ); @@ -1317,6 +1334,20 @@ describe('workflow handoff through teamai-reports', () => { expect(git(bob.checkout, 'rev-parse', `${more.commit}^`)).toBe(wrote.commit); }); + it('hands over a copy that started from the branch of a step it needs, with the branch it started from', async () => { + const acme = team({ stack: STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const runId = await startIn(alice, 'stack'); + const a = await work(alice, runId, 'a.txt'); + const b = await work(alice, runId, 'b.txt'); + + expect(await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout })).toMatchObject({ ok: true, value: { publication: 'published' } }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + const claimed = await readRun(bob.config, runId); + expect(typeof claimed !== 'string' && claimed.copies?.[b.branch]).toEqual({ commit: b.commit, from: a.branch }); + }); + it('refuses to publish, telling to publish again, when the run moves while publish checks the business origin', async () => { const acme = team({ ship: SHIP }); const alice = await acme.member('alice'); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index b816862db..595fa674a 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -265,17 +265,20 @@ function leaseData( } }; } -/** Records who opened a workspace the first time a driver takes its branch, or who first leases a copy a claim brought in. */ +/** + * Records who opened a workspace the first time a driver takes its branch, or who first leases a copy a claim brought in, + * and, before the copy has a commit, the branch it starts from. + */ function rememberCopy(run: RunState, plan: WorkspacePlan, driver: DriverIdentity): void { const branch = plan.contract?.branch; if (!branch) return; - const origin = plan.contract?.start.branch ?? plan.contract?.merge[0]?.branch; + const from = plan.contract?.start.branch ?? plan.contract?.merge[0]?.branch; const copy = (run.copies ??= {})[branch]; if (copy) { copy.owner ??= driver; - if (!copy.origin && !copy.commit && origin && origin !== branch) copy.origin = origin; + if (!copy.from && !copy.commit && from && from !== branch) copy.from = from; } - else run.copies[branch] = { owner: driver, ...(origin ? { origin } : {}) }; + else run.copies[branch] = { owner: driver, ...(from && from !== branch ? { from } : {}) }; } /** diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 2f9c9718a..f3a961d37 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -54,7 +54,7 @@ const BodySchema = z.object({ round: z.number().int().optional(), base: z.object({ ref: z.string().optional(), commit: CommitIdSchema }).optional(), /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ - copies: z.record(z.object({ commit: CommitIdSchema.optional() })), + copies: z.record(z.object({ commit: CommitIdSchema.optional(), from: z.string().optional() })), deliveries: z.array(DeliveryReceiptSchema).optional(), sessionContinuations: z.array(SessionContinuationSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ @@ -75,8 +75,12 @@ const SnapshotSchema = z.object({ write: z.string().min(1), }).merge(BodySchema).superRefine((snapshot, ctx) => { // A claim hands each copy's branch to git, so a snapshot never names a branch outside its own run. - for (const branch of Object.keys(snapshot.copies)) { + for (const [branch, copy] of Object.entries(snapshot.copies)) { if (!branch.startsWith(`teamai/${snapshot.run}/`)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['copies', branch], message: 'is not a branch of this run' }); + // A pull request from the copy may target the branch it started from. + if (copy.from !== undefined && !copy.from.startsWith(`teamai/${snapshot.run}/`)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['copies', branch, 'from'], message: 'is not a branch of this run' }); + } } }); export type RunSnapshot = z.infer; @@ -222,7 +226,7 @@ const FAILURE_RULES = { 'code': 'publish', message: 'rewrite' } satisfies FieldR // A lease another machine held is released when the run is handed over, so the session it offered stays here. const LEASE_RULES = { attempt: 'publish', revision: 'publish', driver: 'rewrite', session: 'drop' } satisfies FieldRules; -const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish' } satisfies FieldRules[string]>; +const COPY_RULES = { owner: 'drop', commit: 'publish', from: 'publish' } satisfies FieldRules[string]>; const FAILURE_REWRITES = { message: () => ({ message: HANDED_FAILURE_MESSAGE }), diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 1171ef134..27ab1571a 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -257,9 +257,10 @@ export interface RunState { checkout?: string; /** * Workspaces this run has handed out, by branch. A `continue` step stays with the driver that opened the copy. A copy a - * claim brought from another machine has no owner until a driver leases it. + * claim brought from another machine has no owner until a driver leases it. `from` is the run branch the copy started + * from, `after` items before other needs, so a pull request from it targets the nearest delivered branch it grew from. */ - copies?: Record; + copies?: Record; /** * The pull requests delivered on this run, once each, in the order they first appeared. Append-only: a receipt says a * request was opened, never that it is still open, so whether a delivery reuses one is its host's lookup to say. From f50d019d85981bea49b4694482eeaadf0728542a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:51:40 +0200 Subject: [PATCH 13/77] feat(workflow): open a pull request into the delivered branch it grew from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delivered branch now targets the nearest branch it started from that delivered a pull request, else the run base branch, so a stacked step's request holds only its own work. The walk follows the `from` each copy keeps, which already orders `after` items before other needs, passes over a skipped need to that need's own needs, and continues a sub-workflow's first step through the workflow step's needs. A reported URL on the origin host is checked against the same target. Tests cover the behaviour already in place too: a composite workflow step opens one request for its shared branch and none for its steps, a looped `new` step opens one per `/p` branch and leaves the earlier ones open, a loop on one `continue` branch lists its request once, and for_each items list theirs in item order. Red: the stacked, `after` item, skipped-need, sub-workflow and composite target tests failed with the second request opened into `main` (`+ "main"` where the needed step's branch was expected); "keeps a URL a stacked step reports …" failed with `expected … to not have property "data.dropped"` while the lookup still used the run base branch. --- src/__tests__/workflow-delivery.test.ts | 143 +++++++++++++++++++++++- src/workflow/engine.ts | 14 ++- src/workflow/workspace.ts | 14 +++ 3 files changed, 162 insertions(+), 9 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 2fd2a3fe5..f96680ffe 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -429,9 +429,146 @@ describe('delivering a step', () => { expect(delivered.ok && delivered.value.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); }); - it.todo('delivers a workflow step with workspace: new once, against the delivered branch of the step it needs'); - it.todo('delivers each for_each item against the delivered branch of the item it waited for, and lists the PRs in item order'); - it.todo('targets the nearest delivered branch, passing over a need that was skipped'); +}); + +/** A command that adds a line no earlier run of it wrote to `file`, so every run commits. */ +const bump = (file: string) => `node -e "require('fs').appendFileSync('${file}', Date.now() + ' ' + Math.random() + '\\n')"`; + +/** The head and target branch of each pull request the fake gh was asked to open, in order. */ +function creates(gh: ReturnType): Array<[string | undefined, string | undefined]> { + return gh.calls().filter((call) => call.args[0] === 'pr').map((call) => [flag(call.args, '-H'), flag(call.args, '-B')]); +} + +/** The commits on `head` that `target` does not have, on the fixture's origin, leaving out the merge that brings a copy up to its needs. */ +function commitsBeyond(root: string, target: string, head: string): string[] { + return git(root, ['--git-dir=remote.git', 'log', '--no-merges', '--format=%H', `${target}..${head}`]).split('\n').filter(Boolean); +} + +function changedTo(run: RunState, id: string): string | undefined { + const step = run.steps.find((item) => item.id === id); + return step && 'changes' in step ? step.changes?.to : undefined; +} + +describe('where a pull request goes', () => { + it('opens the pull request of a step into the delivered branch of the step it needs, with only its own commits', async () => { + const { config, cwd, root, runId } = await fixture(` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n deliver: pr\n`); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [a, b] = [`teamai/${runId}/a/p1`, `teamai/${runId}/b/p1`]; + expect(creates(gh)).toEqual([[a, 'main'], [b, a]]); + const run = await state(config, runId); + expect(commitsBeyond(root, a, b)).toEqual([changedTo(run, 'b')]); + expect(run.deliveries).toEqual([ + { url: 'https://github.com/fixture/app/pull/1', branch: a, base: 'main', step: 'a', pass: 1 }, + { url: 'https://github.com/fixture/app/pull/2', branch: b, base: a, step: 'b', pass: 1 }, + ]); + }); + + it('keeps a URL a stacked step reports when it is the open pull request into the delivered branch of the step it needs', async () => { + const { config, root, runId } = await fixture(` - id: a\n workspace: new\n command: node a.js\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: node b.js\n deliver: pr\n`); + const gh = fakeGh(path.join(root, 'gh')); + const [a, b] = [`teamai/${runId}/a/p1`, `teamai/${runId}/b/p1`]; + const first = await take(config, runId); + await succeed(config, runId, first, { to: pushWork(root, first) }); + const second = await take(config, runId); + gh.configure({ open: [pull(1, a), pull(2, b, a)] }); + + const accepted = await succeed(config, runId, second, { to: pushWork(root, second), pr_url: 'https://github.com/fixture/app/pull/2' }); + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'b', status: 'delivering' } } }); + expect(accepted).not.toHaveProperty('data.dropped'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).deliveries?.[1]).toEqual({ url: 'https://github.com/fixture/app/pull/2', branch: b, base: a, step: 'b', pass: 1 }); + expect(creates(gh)).toEqual([[a, 'main']]); + }); + + it('opens the pull request of a for_each item into the delivered branch of the item it runs after, with only its own commits', async () => { + const { config, cwd, root, runId } = await fixture(` - id: build\n for_each: inputs.packages\n workspace: new\n command: ${bump('items.txt')}\n deliver: pr\n`, + { packages: [{ name: 'one', id: 'one', after: [] }, { name: 'two', id: 'two', after: ['one'] }] }); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [one, two] = [`teamai/${runId}/build-0/p1`, `teamai/${runId}/build-1/p1`]; + expect(creates(gh)).toEqual([[one, 'main'], [two, one]]); + expect(commitsBeyond(root, one, two)).toEqual([changedTo(await state(config, runId), 'build[1]')]); + }); + + it('passes over a need that was skipped, to the delivered branch before it', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: retry\n needs: [a]\n if: steps.a.outcome == 'failed'\n workspace: new\n command: ${bump('retry.txt')}\n deliver: pr\n`, + ` - id: c\n needs: [retry, a]\n workspace: new\n command: ${bump('c.txt')}\n deliver: pr\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ id: 'retry', status: 'skipped' }); + expect(creates(gh)).toEqual([[`teamai/${runId}/a/p1`, 'main'], [`teamai/${runId}/c/p1`, `teamai/${runId}/a/p1`]]); + }); + + it('opens the pull request of a sub-workflow\'s first step into the delivered branch of what the workflow step needs', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: change\n needs: [a]\n workflow:\n steps:\n - id: write\n workspace: new\n command: ${bump('write.txt')}\n deliver: pr\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/a/p1`, 'main'], [`teamai/${runId}/change-write/p1`, `teamai/${runId}/a/p1`]]); + }); + + it('opens one pull request for a workflow step with workspace: new, from the branch its steps share, and none for its steps', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: change\n needs: [a]\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: write\n workspace: continue\n command: ${bump('write.txt')}\n`, + ` - id: more\n needs: [write]\n workspace: continue\n command: ${bump('more.txt')}\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [a, change] = [`teamai/${runId}/a/p1`, `teamai/${runId}/change/p1`]; + expect(creates(gh)).toEqual([[a, 'main'], [change, a]]); + const run = await state(config, runId); + expect(commitsBeyond(root, a, change)).toEqual([changedTo(run, 'change.more'), changedTo(run, 'change.write')]); + expect(run.deliveries?.map((receipt) => [receipt.step, receipt.branch])).toEqual([['a', a], ['change', change]]); + }); + + it('opens a pull request for each pass of a looped new step, leaves the earlier ones open, and lists them in order', async () => { + const { config, cwd, root, runId } = await fixture( + ` - id: fix\n workspace: new\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 2", max_iterations: 2 }\n`, + ); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/fix/p1`, 'main'], [`teamai/${runId}/fix/p2`, 'main']]); + expect(gh.calls().every((call) => call.args[0] === 'api' || (call.args[0] === 'pr' && call.args[1] === 'create'))).toBe(true); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1', 'https://github.com/fixture/app/pull/2']); + }); + + it('opens one pull request for a step that loops on one continue branch, and lists it once', async () => { + const { config, cwd, root, runId } = await fixture( + ` - id: change\n workspace: new\n workflow:\n steps:\n - id: fix\n workspace: continue\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 3", max_iterations: 3 }\n`, + ); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/change/p1`, 'main']]); + expect(gh.calls().filter((call) => call.args[0] === 'api')).toHaveLength(3); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('lists the pull requests of three for_each items in item order', async () => { + const { config, cwd, root, runId } = await fixture(` - id: build\n for_each: inputs.packages\n workspace: new\n command: ${bump('items.txt')}\n deliver: pr\n`, + { packages: ['one', 'two', 'three'].map((name) => ({ name, id: name, after: [] })) }); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([0, 1, 2].map((index) => [`teamai/${runId}/build-${index}/p1`, 'main'])); + const run = await state(config, runId); + expect(run.deliveries?.map((receipt) => receipt.step)).toEqual(['build[0]', 'build[1]', 'build[2]']); + expect(buildEndRecord(run, remote, 'tester').prs).toEqual([1, 2, 3].map((number) => `https://github.com/fixture/app/pull/${number}`)); + }); }); describe('protected paths', () => { diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 595fa674a..a40d0b3b1 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -24,7 +24,7 @@ import { import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { deliverableOf, deliveryProblem, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; import { deliverPr, deliveryFailure, reportedPrMismatch, type DeliveryJob, type DeliveryKey, type DeliveryOutcome } from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; @@ -904,12 +904,13 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { for (const step of run.steps) { if (step.status !== 'delivering') continue; const key = deliveryKey(step); - if (!run.checkout || !run.base?.ref) { + const target = deliveryTarget(run, key.branch); + if (!run.checkout || !run.base || target === undefined) { deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); continue; } return { - key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target: run.base.ref, + key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } @@ -1414,10 +1415,11 @@ function continueSession(run: RunState, report: ReportInput): void { async function reportedPrUnmatched(config: LocalConfig, runId: string, report: ReportInput, url: PrUrl): Promise { if (report.outcome !== 'success' || !report.changes || !isRunId(runId)) return undefined; const run = await readRun(config, runId); - if (typeof run === 'string' || !run.checkout || !run.base?.ref) return undefined; + if (typeof run === 'string' || !run.checkout) return undefined; + const target = deliveryTarget(run, report.changes.branch); const step = run.steps.find((item) => item.status === 'leased' && item.lease.attempt === report.attempt); - if (!step || locateStep(run, step.id)?.step.deliver !== 'pr') return undefined; - return reportedPrMismatch({ checkout: run.checkout, url, branch: report.changes.branch, target: run.base.ref }); + if (!step || target === undefined || locateStep(run, step.id)?.step.deliver !== 'pr') return undefined; + return reportedPrMismatch({ checkout: run.checkout, url, branch: report.changes.branch, target }); } export async function reportAttempt(config: LocalConfig, runId: string, report: ReportInput): Promise { diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 75ee0ea81..80cf36936 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -168,6 +168,20 @@ export function deliverableOf(run: RunState, state: StepState, step: WorkflowSte return branch === undefined || head === undefined || head === run.base?.commit ? undefined : { branch, head }; } +/** + * The branch a pull request from `branch` goes into: the nearest branch it grew from that delivered a pull request, so + * the request holds only the work done since, else the run base branch. Undefined when the run started from no branch. + */ +export function deliveryTarget(run: Pick, branch: string): string | undefined { + const delivered = new Set((run.deliveries ?? []).map((receipt) => receipt.branch)); + const walked = new Set([branch]); + for (let from = run.copies?.[branch]?.from; from !== undefined && !walked.has(from); from = run.copies?.[from]?.from) { + if (delivered.has(from)) return from; + walked.add(from); + } + return run.base?.ref; +} + /** Direct steps of a workflow step, such as `work.write`, not its fan-out items. */ function workflowChildren(run: RunState, step: StepState): StepState[] { const prefix = `${step.id}.`; From c88e9c5d3474f10560d8e3a570579bb9bedcf8b5 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:53:09 +0200 Subject: [PATCH 14/77] fix(workflow): start the next for_each item while an earlier one delivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit max_parallel bounds how many items have an agent at work. An item that finished its agent and waits for its pull request has none, so it no longer holds back the next item. An item that runs after another still waits for it to succeed, so its target is that item's delivered branch. Red: "starts the next for_each item while an earlier one delivers, since max_parallel bounds agents" failed with `no lease: {"data":{"wait":{"reason": "build[0] is delivering",…}}}`. --- src/__tests__/workflow-delivery.test.ts | 14 ++++++++++++++ src/workflow/fan-out.ts | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index f96680ffe..f815c8b78 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -558,6 +558,20 @@ describe('where a pull request goes', () => { expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); }); + it('starts the next for_each item while an earlier one delivers, since max_parallel bounds agents', async () => { + const { config, root, runId } = await fixture(` - id: build\n for_each: { items: inputs.packages, max_parallel: 1 }\n workspace: new\n command: node build.js\n deliver: pr\n`, + { packages: ['one', 'two'].map((name) => ({ name, id: name, after: [] })) }); + const first = await take(config, runId); + await succeed(config, runId, first, { to: pushWork(root, first), pr_url: 'https://review.acme.internal/changes/1' }); + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + try { + expect(await take(config, runId)).toMatchObject({ step: 'build[1]' }); + } finally { + await releaseLock(held); + } + }); + it('lists the pull requests of three for_each items in item order', async () => { const { config, cwd, root, runId } = await fixture(` - id: build\n for_each: inputs.packages\n workspace: new\n command: ${bump('items.txt')}\n deliver: pr\n`, { packages: ['one', 'two', 'three'].map((name) => ({ name, id: name, after: [] })) }); diff --git a/src/workflow/fan-out.ts b/src/workflow/fan-out.ts index cd1662fcb..23701ae0e 100644 --- a/src/workflow/fan-out.ts +++ b/src/workflow/fan-out.ts @@ -93,9 +93,9 @@ export function afterStanding(after: StepState[]): 'ready' | 'waiting' | 'skip' return after.every((item) => item.status === 'succeeded') ? 'ready' : 'waiting'; } -/** How many items are started and not yet settled, which `max_parallel` bounds. */ +/** How many items have an agent or workflow at work, which `max_parallel` bounds. An item that delivers has none. */ export function startedItems(items: StepState[]): number { - return items.filter((item) => item.status === 'leased' || item.status === 'running' || item.status === 'delivering').length; + return items.filter((item) => item.status === 'leased' || item.status === 'running').length; } /** From 1c70b3052b547e3b328a274eb46eb44dce6a4127 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 12:58:41 +0200 Subject: [PATCH 15/77] feat(workflow): record a pull request a stopped delivery left open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate marks a step as creating right before TeamAI asks the host to open a request. When a cancel lands during the create and the process stops before it hears back, the host holds a request the run never lists. The drain in `next` now also looks up every step that settled otherwise with that mark and no receipt for its pass, on an ended run too. It only looks up and never opens one. A request it finds gets its receipt, and an ended run queues its end record again to list it. An answer either way drops the mark. A lookup that fails keeps the mark for a later call and is tried once per drain. Tests also pin what already held: retrying a failed delivery delivers again in the same pass and attempt with no lease, retrying a protected-path failure runs the step again in a new pass, cancelling a delivering step keeps its branch on origin, a cancel during the lookup opens nothing, and a request opened during a cancel is recorded with the step left cancelled. Red: "records a pull request a cancelled delivery opened before its process stopped, by looking it up on a later call, and opens no other" failed with `expected undefined to deeply equal [ { …(5) } ]`. The later `next` left the ended run without a receipt. --- src/__tests__/workflow-delivery.test.ts | 131 +++++++++++++++++++++++- src/workflow/engine.ts | 50 +++++++-- 2 files changed, 172 insertions(+), 9 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index f815c8b78..192285759 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -19,6 +19,9 @@ import { buildEndRecord, pendingRecords } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; import * as worktree from '../workflow/worktree.js'; import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import * as registry from '../providers/registry.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import type { GitProvider, PullRequestLookup } from '../providers/types.js'; import type { LocalConfig } from '../types.js'; import type { RunState } from '../workflow/types.js'; @@ -68,7 +71,7 @@ async function state(config: LocalConfig, runId: string): Promise { } const LeaseSchema = z.object({ - step: z.string(), attempt: z.string(), revision: z.number(), + step: z.string(), attempt: z.string(), revision: z.number(), pass: z.number(), workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), }); type Lease = z.infer; @@ -654,3 +657,129 @@ describe('reported URLs from a driver', () => { expect(gh.calls().filter((call) => call.args[0] === 'pr')).toEqual([]); }); }); + +/** + * Puts a provider double on the fixture's GitHub origin: `findOpen` answers lookups and `create` opens requests. Returns + * the create spy, so a test can show what was asked of the host. + */ +function providerDouble(findOpen: PullRequestLookup['findOpen'], create: GitProvider['createPullRequest']) { + const provider = new GitHubProvider(); + const creates = vi.spyOn(provider, 'createPullRequest').mockImplementation(create); + vi.spyOn(registry, 'providerForRemote').mockReturnValue({ kind: 'provider', provider, pullRequests: { findOpen }, repo: 'fixture/app', hostname: 'github.com' }); + return creates; +} + +describe('retrying and cancelling a delivery', () => { + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + + gh.configure({}); + expect(await retryStep(config, runId, 'fix')).toMatchObject({ data: {} }); + const retried = (await state(config, runId)).steps[0]; + expect(retried).toMatchObject({ id: 'fix', status: 'delivering', attempt: fix.attempt }); + expect(retried?.pass ?? 1).toBe(1); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(creates(gh)).toEqual([[`teamai/${runId}/fix/p1`, 'main']]); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', attempt: fix.attempt }); + }); + + it('retries a branch that changed protected paths by running the step again, in a new pass', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n' }) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + + expect(await retryStep(config, runId, 'fix')).toMatchObject({ data: {} }); + const again = await take(config, runId); + expect(again).toMatchObject({ step: 'fix', pass: 2, workspace: { branch: `teamai/${runId}/fix/p2` } }); + expect(again.attempt).not.toBe(fix.attempt); + expect(gh.calls()).toEqual([]); + }); + + it('cancels a delivering step and keeps its branch on origin', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await cancelRun(config, runId, 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'] } } }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/teamai/${runId}/fix/p1`])).toBe(to); + expect(gh.calls()).toEqual([]); + }); + + it('opens nothing when the run is cancelled while its delivery looks for an open pull request', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const create = providerDouble(async () => { + await cancelRun(config, runId); + return undefined; + }, async () => 'https://github.com/fixture/app/pull/1'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it('records a pull request opened while the run was cancelled, keeps the step cancelled, and queues the end record again with it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + providerDouble(async () => undefined, async () => { + await cancelRun(config, runId); + return url; + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch: `teamai/${runId}/fix/p1`, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + }); + + it('records a pull request a cancelled delivery opened before its process stopped, by looking it up on a later call, and opens no other', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + const open: string[] = []; + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + await cancelRun(config, runId); + open.push(url); + opened(); + // The process stops here: the host opened the request, and the run never hears of it. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + // A stopped process leaves its delivery lock to whoever comes next. + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch, creating: true } }); + expect((await state(config, runId)).deliveries).toBeUndefined(); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + expect(create).toHaveBeenCalledTimes(1); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + expect((await state(config, runId)).deliveries).toHaveLength(1); + }); +}); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index a40d0b3b1..08ca3c7fa 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -917,6 +917,31 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { return undefined; } +/** + * A request a step asked its host to open before it settled otherwise, cancelled or failed, with no receipt for its pass: + * the host may hold one the run does not list. Its job is only looked up, never opened. `tried` holds the steps this + * drain already looked up, so one the host cannot answer for now waits for a later call. + */ +function unresolvedCreation(run: RunState, tried: ReadonlySet): DeliveryJob | undefined { + if (!run.checkout || !run.base) return undefined; + for (const step of run.steps) { + if (step.status === 'delivering' || !('delivery' in step) || !step.delivery?.creating || tried.has(step.id)) continue; + const key = { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; + if (run.deliveries?.some((receipt) => receipt.step === key.step && receipt.pass === key.pass && receipt.branch === key.branch)) continue; + const target = deliveryTarget(run, key.branch); + if (target !== undefined) return { key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target }; + } + return undefined; +} + +/** Drops the mark of the request `key` asked for, once a lookup said whether its host holds it. */ +function creationResolved(run: RunState, key: DeliveryKey): void { + const step = run.steps.find((item) => item.id === key.step && (item.pass ?? 1) === key.pass); + if (step && step.status !== 'delivering' && 'delivery' in step && step.delivery?.branch === key.branch && step.delivery.head === key.head) { + delete step.delivery.creating; + } +} + /** * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The step settles only @@ -953,29 +978,38 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob } /** - * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it. The run lock is - * held to pick a delivery and to apply its outcome, never across git or a provider. While another process of this - * machine delivers, it holds the delivery lock, and this call leaves the work to it. + * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it, then looks up + * the requests that steps which stopped delivering may have left open, on an ended run too. The run lock is held to pick + * a delivery and to apply its outcome, never across git or a provider. While another process of this machine delivers, + * it holds the delivery lock, and this call leaves the work to it. */ async function drainDeliveries(config: LocalConfig, runId: string): Promise { if (!isRunId(runId)) return; const loaded = await readRun(config, runId); - if (typeof loaded === 'string' || loaded.status !== 'running' || !loaded.steps.some((step) => step.status === 'delivering')) return; + if (typeof loaded === 'string') return; + const delivering = loaded.status === 'running' && loaded.steps.some((step) => step.status === 'delivering'); + if (!delivering && !unresolvedCreation(loaded, new Set())) return; const drained = await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { let requeued = false; + const tried = new Set(); for (;;) { const picked = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); - const job = nextDelivery(run); - return { value: job, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; + const next = nextDelivery(run); + const unresolved = next ? undefined : unresolvedCreation(run, tried); + const value = next ? { job: next, lookupOnly: false } : unresolved && { job: unresolved, lookupOnly: true }; + return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); if (picked.status !== 'done' || !picked.value) return requeued; - const job = picked.value; - const outcome = await deliverPr(job, () => markCreating(config, runId, job)); + const { job, lookupOnly } = picked.value; + if (lookupOnly) tried.add(job.key.step); + // A gate that never opens leaves a lookup: the request is found, or `withdrawn` says the host holds none. + const outcome = await deliverPr(job, lookupOnly ? async () => false : () => markCreating(config, runId, job)); const applied = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const { requeueEnd } = completeDelivery(run, job, outcome); + if (lookupOnly && (outcome.ok || outcome.fault.kind === 'withdrawn')) creationResolved(run, job.key); // Queued under the lock with the receipt, as finishRun queues an end; it publishes once the lock is released. if (requeueEnd) run.endRecordQueued = await queueRunRecord(config, run, run.driver, 'end'); return { value: requeueEnd, write: { ...run, updatedAt: new Date().toISOString() } }; From 1bc6cb99c9164bb4f6c037c8b2db96a11702cd5a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:04:37 +0200 Subject: [PATCH 16/77] test(workflow): hand over a run while a step delivers its pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run published while a step delivers carries what it delivers, the branch its copy started from and the receipts so far. The install that claims it resumes the delivery through the provider's lookup. When the first machine already opened the request, the claimer records it and opens none. When none exists yet, the claimer opens exactly one, into the delivered branch the copy grew from. A snapshot whose receipt or reported URL is not one TeamAI keeps is refused at the claim, naming the field. No code change was needed: phase one's snapshot rules and the lineage fix in the previous commits give this behaviour. These tests hold it. Red: with the lineage fix reverse-applied, both resume tests failed at publish (`expected { ok: false, error: { …(3) } } to match object { ok: true, … }`). With the snapshot schemas parsing receipt and reported URLs as plain strings, both URL refusal tests failed with the claim accepted (`expected { ok: true, … } to match object { ok: false, … }`). --- src/__tests__/workflow-handoff.test.ts | 108 +++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/src/__tests__/workflow-handoff.test.ts b/src/__tests__/workflow-handoff.test.ts index d5cdf8e41..37b119383 100644 --- a/src/__tests__/workflow-handoff.test.ts +++ b/src/__tests__/workflow-handoff.test.ts @@ -22,6 +22,7 @@ import type { StepPass, StepState } from '../workflow/step-state.js'; import { readRun, runLockPath, updateRun } from '../workflow/store.js'; import type { DriverIdentity, RunState, WorkflowEnvelope } from '../workflow/types.js'; import { ensureWorktree, fetchMissingCommits, headCommit, markReadCopy, worktreePath } from '../workflow/worktree.js'; +import { fakeGh } from './helpers/fake-gh.js'; const dirs: string[] = []; @@ -105,6 +106,21 @@ steps: command: echo b > b.txt `; +const DELIVERED_STACK = `schemaVersion: v1alpha1 +name: stack +description: A change delivered as a pull request, and one stacked on it. +steps: + - id: a + workspace: new + command: echo a > a.txt + deliver: pr + - id: b + needs: [a] + workspace: new + command: echo b > b.txt + deliver: pr +`; + const WORKSPACES = ['read-only', 'new', 'continue'] as const; interface Member { @@ -333,6 +349,38 @@ async function runStep(member: Member, runId: string, outputs?: Record): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr').map((call) => ['-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +/** + * Starts the delivered stack on `alice`, delivers `a` and reports `b`, so `b` is delivering when the run is published. + * The fake host keeps `a`'s pull request. + */ +async function deliveringStack(acme: ReturnType, alice: Member, bob: Member) { + onGitHub(acme.business, [alice, bob]); + const gh = fakeGh(path.join(path.dirname(acme.origin), 'gh')); + const runId = await startIn(alice, 'stack'); + const a = await work(alice, runId, 'a.txt'); + const b = await work(alice, runId, 'b.txt'); + return { gh, runId, a, b }; +} + describe('workflow handoff through teamai-reports', () => { it('hands a published run to another install, which continues it, and refuses the earlier owner', async () => { const acme = team(); @@ -1348,6 +1396,66 @@ describe('workflow handoff through teamai-reports', () => { expect(typeof claimed !== 'string' && claimed.copies?.[b.branch]).toEqual({ commit: b.commit, from: a.branch }); }); + it('publishes a step that delivers with what it delivers, and the claim finds the pull request the first machine opened and opens no other', async () => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { gh, runId, a, b } = await deliveringStack(acme, alice, bob); + // Alice's process opened b's request too, then stopped before the run heard of it. + gh.configure({ open: [openPull(1, a.branch), openPull(2, b.branch, a.branch)] }); + + expect(await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout })).toMatchObject({ ok: true, value: { publication: 'published' } }); + expect(JSON.parse(originFile(acme.origin, `workflows/runs/${runId}/state.json`) ?? 'null')).toMatchObject({ + copies: { [b.branch]: { commit: b.commit, from: a.branch } }, + deliveries: [{ url: 'https://github.com/acme/app/pull/1', branch: a.branch, base: 'main', step: 'a', pass: 1 }], + steps: [{ id: 'a', status: 'succeeded' }, { id: 'b', status: 'delivering', delivery: { branch: b.branch, head: b.commit } }], + }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + + expect(await leaseNext(bob.config, runId, bob.driver)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await readRun(bob.config, runId); + expect(typeof run !== 'string' && run.steps[1]).toMatchObject({ id: 'b', status: 'succeeded' }); + expect(typeof run !== 'string' && run.deliveries?.[1]).toEqual({ url: 'https://github.com/acme/app/pull/2', branch: b.branch, base: a.branch, step: 'b', pass: 1 }); + expect(creates(gh)).toEqual([[a.branch, 'main']]); + }); + + it('opens the pull request of a step that was delivering when the run was claimed, once, on the machine that claimed it', async () => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { gh, runId, a, b } = await deliveringStack(acme, alice, bob); + await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + expect(creates(gh)).toEqual([[a.branch, 'main']]); + + expect(await leaseNext(bob.config, runId, bob.driver)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[a.branch, 'main'], [b.branch, a.branch]]); + const run = await readRun(bob.config, runId); + expect(typeof run !== 'string' && run.deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/acme/app/pull/1', 'https://github.com/acme/app/pull/2']); + }); + + it.each<[string, (snapshot: { deliveries: Array<{ url: string }>; steps: Array<{ delivery?: { reported?: string } }> }) => void, string]>([ + ['a receipt', (snapshot) => { snapshot.deliveries = snapshot.deliveries.map((receipt) => ({ ...receipt, url: 'https://user:token@github.com/acme/app/pull/1' })); }, 'deliveries[0].url'], + ['a reported URL', (snapshot) => { const step = snapshot.steps[1]; if (step?.delivery) step.delivery.reported = 'http://github.com/acme/app/pull/2'; }, 'steps[1].delivery.reported'], + ])('refuses to claim a snapshot whose pull request URL in %s is not one TeamAI keeps, naming the field', async (_name, edit, field) => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { runId } = await deliveringStack(acme, alice, bob); + await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout }); + const editor = path.join(path.dirname(acme.origin), 'editor'); + git(path.dirname(acme.origin), 'clone', '-q', '--branch', 'teamai-reports', acme.origin, editor); + const rel = `workflows/runs/${runId}/state.json`; + const snapshot = JSON.parse(fs.readFileSync(path.join(editor, rel), 'utf8')); + edit(snapshot); + fs.writeFileSync(path.join(editor, rel), `${JSON.stringify(snapshot, null, 2)}\n`); + git(editor, 'commit', '-qam', 'hand edit'); + git(editor, 'push', '-q', 'origin', 'teamai-reports'); + + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field } }); + expect(await readRun(bob.config, runId)).toBe('missing'); + }); + it('refuses to publish, telling to publish again, when the run moves while publish checks the business origin', async () => { const acme = team({ ship: SHIP }); const alice = await acme.member('alice'); From 4eae92b82bb833e3c6396b5786253be016fdde5b Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:08:24 +0200 Subject: [PATCH 17/77] feat(workflow): judge the agent pass of a step that delivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scorers judge what the agent did, so delivery no longer decides whether, or by which install, a pass is judged. A delivering pass is judgeable by the install that reported it, as it will be once delivered. A pass whose only failure is delivery_failed or protected_path counts as the agent's success, also once a retry moved it into the step's history. A run claimed mid-delivery is therefore judged by the reporting install. Before, its pass was not judgeable there while delivering, and never on the claimer. The evidence for such a pass reads the same agent pass. A label made while the step delivered stays the pass's only judgement whether delivery then succeeds or fails. Scoring stays pull-only: the engine never calls it, so delivery never waits on a scorer. The failure codes only delivery writes are now a type that `deliveryFailed` and the delivery failure builder share, so scoring's list cannot drift from what delivery writes. Red: "judges the agent pass of a step that delivers, for the install that reported it, whatever becomes of its delivery" failed with `expected [] to deeply equal [ { subject: { …(3) }, …(1) } ]`. "judges a step while it delivers, and its label stays the judgement however the delivery ends" failed with no lease offered (ZodError: lease Required). After the scorer fix it still failed: the entry closed as `evidence_empty` until the evidence plan read the agent pass. --- src/__tests__/workflow-scoring.test.ts | 50 +++++++++++++++++++++++++- src/workflow/loop.ts | 6 ++-- src/workflow/pr-delivery.ts | 4 +-- src/workflow/scoring-evidence.ts | 4 +-- src/workflow/scoring.ts | 12 ++++--- src/workflow/step-state.ts | 20 ++++++++++- 6 files changed, 83 insertions(+), 13 deletions(-) diff --git a/src/__tests__/workflow-scoring.test.ts b/src/__tests__/workflow-scoring.test.ts index a9cb11ece..f010a5163 100644 --- a/src/__tests__/workflow-scoring.test.ts +++ b/src/__tests__/workflow-scoring.test.ts @@ -16,7 +16,8 @@ import { workflowRunDetail } from '../dashboard/workflow-runs.js'; import { buildEndRecord } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; import { nextJudgeLease, parseJudgeRequest, parseJudgeSubmission, runJudgeLease, scoreOnDemand, scoredPasses, submitJudgeLease, type JudgeRequest } from '../workflow/scoring.js'; -import { installOwner, readRun } from '../workflow/store.js'; +import { installOwner, readRun, updateRun } from '../workflow/store.js'; +import type { StepState } from '../workflow/step-state.js'; import type { DriverIdentity, RunState, WorkflowEnvelope } from '../workflow/types.js'; const dirs: string[] = []; @@ -98,7 +99,32 @@ async function settledRun(config: LocalConfig, reported: Record return runId; } +/** Rewrites run `runId`'s one step as `step`, with the run still running, as a delivery would leave it. */ +async function setStep(config: LocalConfig, runId: string, step: (run: RunState) => StepState): Promise { + await updateRun(config, runId, async (run) => (run === 'missing' ? { value: undefined } : { value: undefined, write: { ...run, status: 'running', steps: [step(run)] } })); +} + describe('judge work', () => { + it('judges a step while it delivers, and its label stays the judgement however the delivery ends', async () => { + const config = fixture(scorer(100)); + const runId = await settledRun(config); + const settled = (await runOf(config, runId)).steps[0]; + if (settled?.status !== 'succeeded') throw new Error('review did not succeed'); + const reported = (run: RunState) => ({ ...settled, delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } }); + await setStep(config, runId, (run) => ({ ...reported(run), status: 'delivering' })); + + const lease = leaseOf(await nextJudgeLease(config, request())); + data(await submitJudgeLease(config, lease.ref, { driver: judgeDriver, outcome: 'success', answer: { label: 'good', reasoning: 'r' } })); + const labels = await LabelStore.local(config); + expect(labels).toMatchObject([{ subject: { runId, step: 'review', pass: 1 }, label: 'good' }]); + + await setStep(config, runId, (run) => ({ ...reported(run), status: 'failed', outcome: 'failed', failure: { code: 'delivery_failed', message: 'm' } })); + expect(data(await nextJudgeLease(config, request()))).not.toHaveProperty('lease'); + await setStep(config, runId, (run) => ({ ...reported(run), status: 'succeeded', outcome: 'success' })); + expect(data(await nextJudgeLease(config, request()))).not.toHaveProperty('lease'); + expect(await LabelStore.local(config)).toEqual(labels); + }); + it('finalizes the selected Codex tool in the lease', async () => { const config = fixture(scorer(100)); await settledRun(config); @@ -494,6 +520,28 @@ describe('scoring totals', () => { }); describe('scoredPasses', () => { + it('judges the agent pass of a step that delivers, for the install that reported it, whatever becomes of its delivery', async () => { + const config = fixture(scorer(100)); + const run = await runOf(config, await settledRun(config)); + const install = (await installOwner(config)).id; + const reported = { id: 'review', settledAt: 2, attempt: 'a1', installId: install, delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } }; + const judged = [{ subject: { runId: run.id, step: 'review', pass: 1 }, scorers: ['quality'] }]; + const withStep = (step: StepState): RunState => ({ ...run, status: 'running', steps: [step] }); + + expect(scoredPasses(withStep({ ...reported, status: 'delivering' }), install)).toEqual(judged); + expect(scoredPasses(withStep({ ...reported, status: 'succeeded', outcome: 'success' }), install)).toEqual(judged); + for (const code of ['delivery_failed', 'protected_path']) { + expect(scoredPasses(withStep({ ...reported, status: 'failed', outcome: 'failed', failure: { code, message: 'm' } }), install)).toEqual(judged); + const retried: StepState = { id: 'review', status: 'pending', pass: 2, passes: [{ pass: 1, outcome: 'failed', failure: { code, message: 'm' }, installId: install }] }; + expect(scoredPasses(withStep(retried), install)).toEqual(judged); + } + expect(scoredPasses(withStep({ id: 'review', status: 'failed', outcome: 'failed', settledAt: 2, installId: install, failure: { code: 'tool_failed', message: 'm' } }), install)).toEqual([]); + // Claimed mid-delivery: the install that reported the pass judges it, the one that claimed the run does not. + const claimed = { ...withStep({ ...reported, status: 'delivering' }), publication: { stateRevision: 2 } }; + expect(scoredPasses(claimed, install)).toEqual(judged); + expect(scoredPasses(claimed, 'claimer')).toEqual([]); + }); + it('keeps only the passes this install settled, and attributes archived children to the workflow pass that held them', async () => { const config = fixture(scorer(100)); const run = await runOf(config, await settledRun(config)); diff --git a/src/workflow/loop.ts b/src/workflow/loop.ts index df00e5fab..0d23314fd 100644 --- a/src/workflow/loop.ts +++ b/src/workflow/loop.ts @@ -236,11 +236,11 @@ export function archiveUnder(run: RunState, path: string): void { /** A pass of the run, current or archived, with the path of the instance it belongs to. */ export type RunPass = StepPass & { step: string; within?: { step: string; pass: number } }; -/** Every pass of the run, current and archived. */ -export function runPasses(run: RunState): RunPass[] { +/** Every pass of the run, current and archived, with each step's current pass as `current` reads it. */ +export function runPasses(run: RunState, currentPass: (state: StepState) => StepPass | undefined = passOf): RunPass[] { return [ ...run.steps.flatMap((state) => { - const current = passOf(state); + const current = currentPass(state); return [...state.passes ?? [], ...(current ? [current] : [])].map((pass) => ({ ...pass, step: state.id })); }), ...(run.archive ?? []), diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index bb340a162..75bad2ad0 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -7,7 +7,7 @@ import os from 'node:os'; import { providerForRemote, type OriginProvider } from '../providers/registry.js'; import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; -import type { WorkflowFailure } from './types.js'; +import type { DeliveryFailure } from './step-state.js'; import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; /** Which delivery an outcome belongs to: it applies only while the step still delivers this pass of this commit. */ @@ -148,7 +148,7 @@ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promis const LISTED_FILES = 20; /** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ -export function deliveryFailure({ runId, key }: Pick, fault: DeliveryFault): WorkflowFailure { +export function deliveryFailure({ runId, key }: Pick, fault: DeliveryFault): DeliveryFailure { const { step, branch } = key; const retry = `teamai workflow retry ${runId} ${step}`; switch (fault.kind) { diff --git a/src/workflow/scoring-evidence.ts b/src/workflow/scoring-evidence.ts index 3e2a3c0c5..235c0704c 100644 --- a/src/workflow/scoring-evidence.ts +++ b/src/workflow/scoring-evidence.ts @@ -13,7 +13,7 @@ import { import type { RunSubject } from '../scorers/label.js'; import { firstSettled, isUnder, passRange, passesWithin, runPasses } from './loop.js'; import { stepDefinitions } from './scope.js'; -import type { StepPass } from './step-state.js'; +import { agentPassOf, type StepPass } from './step-state.js'; import type { RunState } from './types.js'; import { branchName } from './workspace.js'; import { stepCheckout } from './worktree.js'; @@ -29,7 +29,7 @@ function ranIn(run: RunState, pass: StepPass): string | undefined { export function evidencePlan(run: RunState, subject: RunSubject) { const definitionOf = stepDefinitions(run); - const all = runPasses(run); + const all = runPasses(run, agentPassOf); const own = all.find((pass) => pass.step === subject.step && pass.pass === subject.pass); const workflow = definitionOf(subject.step)?.workflow !== undefined; const gaps: EvidenceGap[] = []; diff --git a/src/workflow/scoring.ts b/src/workflow/scoring.ts index 480e22331..4b46d1fea 100644 --- a/src/workflow/scoring.ts +++ b/src/workflow/scoring.ts @@ -35,6 +35,7 @@ import { error, ok } from './engine.js'; import { localDriver } from './local-driver.js'; import { firstSettled, passesWithin, runPasses, type RunPass } from './loop.js'; import { stepDefinitions } from './scope.js'; +import { agentPassOf, agentSucceeded } from './step-state.js'; import { parseStepPath } from './step-path.js'; import { installOwner, isRunId, readRun, runsDir } from './store.js'; import type { InlineWorkflow, RunState, WorkflowDefinition, WorkflowEnvelope, WorkflowStep } from './types.js'; @@ -50,12 +51,15 @@ function settledHere(run: RunState, pass: RunPass, install: string): boolean { return pass.installId === undefined ? run.publication === undefined : pass.installId === install; } -/** The passes a judge can judge: succeeded passes of prompt and workflow steps, and of a `for_each` step only its items. */ +/** + * The passes a judge can judge: passes of prompt and workflow steps whose agent succeeded, delivering or not, and of a + * `for_each` step only its items. + */ function judgeablePasses(run: RunState): Array<{ pass: RunPass; definition: WorkflowStep }> { const definitionOf = stepDefinitions(run); - return runPasses(run).flatMap((pass) => { + return runPasses(run, agentPassOf).flatMap((pass) => { const definition: WorkflowStep | undefined = definitionOf(pass.step); - if (pass.outcome !== 'success' || !definition) return []; + if (!agentSucceeded(pass) || !definition) return []; if (definition.forEach && parseStepPath(pass.step).at(-1)?.index === undefined) return []; return [{ pass, definition }]; }); @@ -67,7 +71,7 @@ function judgeablePasses(run: RunState): Array<{ pass: RunPass; definition: Work * stamp. Sampling is left to the caller. */ export function scoredPasses(run: RunState, install: string): ScorablePass[] { - const all = runPasses(run); + const all = runPasses(run, agentPassOf); const pinned = run.scorers ?? {}; return judgeablePasses(run).flatMap(({ pass, definition }) => { const scorers = (definition.scorers?.names ?? []).filter((name) => Object.hasOwn(pinned, name)); diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index d2069f6f1..6036f0b42 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -306,8 +306,12 @@ export function delivered(state: DeliveringStep): SucceededStep { return succeed(state, settlement); } +/** The failure codes only delivery gives a step, always after its agent succeeded. */ +const DELIVERY_FAILURE_CODES = ['delivery_failed', 'protected_path'] as const; +export type DeliveryFailure = { code: (typeof DELIVERY_FAILURE_CODES)[number]; message: string }; + /** A delivery that failed stops the step, keeping what its agent did and the branch a retry delivers again. */ -export function deliveryFailed(state: DeliveringStep, failure: WorkflowFailure): FailedStep { +export function deliveryFailed(state: DeliveringStep, failure: DeliveryFailure): FailedStep { const { id: _id, status: _status, ...settlement } = state; return fail(state, { ...settlement, message: failure.message, failure }); } @@ -341,6 +345,20 @@ export function passOf(state: StepState): StepPass | undefined { return StepPassSchema.parse({ ...state, pass: state.pass ?? 1, outcome }); } +/** + * The current pass of a step as its agent left it, which scorers judge: a step that delivers ran its agent to success, + * so delivery never decides whether, or by which install, a pass is judged. + */ +export function agentPassOf(state: StepState): StepPass | undefined { + return state.status === 'delivering' ? StepPassSchema.parse({ ...state, pass: state.pass ?? 1, outcome: 'success' }) : passOf(state); +} + +/** Whether the agent of `pass` succeeded: it did, or only the delivery of what it wrote failed. */ +export function agentSucceeded(pass: StepPass): boolean { + const code: string | undefined = pass.failure?.code; + return pass.outcome === 'success' || (pass.outcome === 'failed' && DELIVERY_FAILURE_CODES.some((delivery) => delivery === code)); +} + /** * A loop runs a settled step again: its current pass joins `passes`, and it waits as the next pass, with nothing of * the workflow it ran. `iteration` is the runs a loop owner has had so far. From 5dcf97fab7dde69e2ee78670c42e909ece3620f3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:11:56 +0200 Subject: [PATCH 18/77] feat(providers): look up the open GitLab merge request of a delivered branch Workflow delivery on a GitLab origin now finds the open merge request of a branch before it opens one, so a delivery that ran before, or ran on another machine, opens no second request. The lookup asks `/projects/:id/merge_requests` for the opened requests of the source branch into the target, with a timeout. It parses the answer at the boundary and keeps only a request from the project itself, not a fork with the same branch name. A lookup that cannot tell is a fault, never "none", since a false "none" opens a duplicate: no token, an HTTP error, an answer it cannot read, a full page without the branch, a URL on another host than the instance, or no answer in time. Its errors never repeat the host's text. Create already honours the delivery timeout. A test now pins that too. Red: both lookup tests failed with `gitlab has no pull request lookup`, and the origin test for `git@gitlab.com:acme/platform/app.git` failed with `expected false to deeply equal [ 'gitlab', 'acme/platform/app' ]`. --- src/__tests__/workflow-pr-providers.test.ts | 82 +++++++++++++++++++-- src/providers/gitlab/gitlab-api.ts | 48 ++++++++++++ src/providers/gitlab/index.ts | 6 +- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 3c0ed19d6..c8602c47a 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -1,11 +1,12 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { GitHubProvider } from '../providers/github/index.js'; +import { GitLabProvider } from '../providers/gitlab/index.js'; import { GenericGitProvider } from '../providers/git/index.js'; import { providerForRemote } from '../providers/registry.js'; -import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; +import type { GitProvider, PullRequestLookup, PullRequestRef } from '../providers/types.js'; import { fakeGh, resolveGh } from './helpers/fake-gh.js'; const ref: PullRequestRef = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', timeoutMs: 1000 }; @@ -61,14 +62,81 @@ describe('GitHub pull request lookup', () => { }); }); +function lookupOf(provider: GitProvider): PullRequestLookup { + if (!provider.pullRequests) throw new Error(`${provider.name} has no pull request lookup`); + return provider.pullRequests; +} + +/** Answers every request with `body` as JSON and `status`, and returns the spy. */ +function answering(body: unknown, status = 200) { + const fetcher = vi.fn().mockImplementation(async () => new Response(JSON.stringify(body), { status })); + vi.stubGlobal('fetch', fetcher); + return fetcher; +} + +/** Answers no request until its signal aborts it, as a host that never responds. */ +function hanging(): void { + vi.stubGlobal('fetch', vi.fn((_url, init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }))); +} + +/** An open merge request as the GitLab API lists it. */ +const mr = (source: string, iid = 7, sourceProject = 1, url = `https://gitlab.com/acme/app/-/merge_requests/${iid}`) => ({ + web_url: url, state: 'opened', source_branch: source, target_branch: 'main', source_project_id: sourceProject, target_project_id: 1, +}); + +describe('GitLab merge request lookup', () => { + beforeEach(() => { + vi.stubEnv('GITLAB_TOKEN', 'test-token'); + vi.stubEnv('GITLAB_URL', ''); + vi.stubEnv('TEAMAI_GITLAB_HOST', ''); + }); + + it('finds the open request of this project branch into the target, never one from a fork with the same branch name', async () => { + const fetcher = answering([mr(ref.source, 6, 2), mr('other', 8), mr(ref.source)]); + + expect(await lookupOf(new GitLabProvider()).findOpen(ref)).toBe('https://gitlab.com/acme/app/-/merge_requests/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('source_branch'), url.searchParams.get('target_branch')]) + .toEqual(['https://gitlab.com/api/v4/projects/acme%2Fapp/merge_requests', 'opened', ref.source, 'main']); + answering([mr(ref.source, 6, 2)]); + expect(await lookupOf(new GitLabProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, without a token, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new GitLabProvider()); + vi.stubEnv('GITLAB_TOKEN', ''); vi.stubEnv('GITLAB_PRIVATE_TOKEN', ''); vi.stubEnv('GITLAB_PAT', ''); + await expect(lookup.findOpen(ref)).rejects.toThrow('GitLab authentication unavailable'); + vi.stubEnv('GITLAB_TOKEN', 'test-token'); + answering({ message: 'private-token-in-body' }, 403); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitLab answered the open merge request lookup with HTTP 403\.(?!.*private-token-in-body)/); + answering({ message: 'private-token-in-body' }); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitLab answered the open merge request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => mr(`other-${index}`, index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open merge requests than one page'); + answering([mr(ref.source, 7, 1, 'https://evil.example/acme/app/-/merge_requests/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); + + it('gives up on a create GitLab does not answer in time', async () => { + hanging(); + await expect(new GitLabProvider().createPullRequest({ repo: 'acme/app', source: ref.source, target: 'main', title: 't', spawnTimeoutMs: 20 })).rejects.toThrow(); + }); +}); + describe('the provider of a checkout origin', () => { it.each([ - ['https://user:secret@github.com/acme/app.git', 'acme/app'], - ['ssh://git@github.com/acme/app.git', 'acme/app'], - ['git@github.com:acme/app.git', 'acme/app'], - ])('reads GitHub and the repository from %s', (remote, repo) => { + ['https://user:secret@github.com/acme/app.git', 'github', 'acme/app'], + ['ssh://git@github.com/acme/app.git', 'github', 'acme/app'], + ['git@github.com:acme/app.git', 'github', 'acme/app'], + ['git@gitlab.com:acme/platform/app.git', 'gitlab', 'acme/platform/app'], + ])('reads the provider and the repository from %s', (remote, provider, repo) => { + vi.stubEnv('GITLAB_URL', ''); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); const origin = providerForRemote(remote); - expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual(['github', repo]); + expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual([provider, repo]); }); it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme'])( diff --git a/src/providers/gitlab/gitlab-api.ts b/src/providers/gitlab/gitlab-api.ts index a2a426cc4..d5e252d8c 100644 --- a/src/providers/gitlab/gitlab-api.ts +++ b/src/providers/gitlab/gitlab-api.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process'; +import { z } from 'zod'; import { GITLAB_HOST } from './repo-url.js'; import { sanitizeGitUrl } from '../../utils/redact.js'; +import type { PullRequestRef } from '../types.js'; /** * GitLab REST API client (API v4). @@ -368,3 +370,49 @@ export async function gitlabMrCreate(opts: GitLabMrCreateOptions): Promise { + const token = requireToken(); + const query = new URLSearchParams({ state: 'opened', source_branch: source, target_branch: target, per_page: String(LOOKUP_PAGE) }); + const response = await fetch(`${gitlabApiBase()}/projects/${encodeURIComponent(repo)}/merge_requests?${query}`, { + headers: authHeaders(token), redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`GitLab answered the open merge request lookup with HTTP ${response.status}. Check the token's access to the project.`); + const requests = z.array(GitLabOpenMrSchema).safeParse(await response.json().catch(() => undefined)); + if (!requests.success) throw new Error('GitLab answered the open merge request lookup with data TeamAI cannot read.'); + const found = requests.data.find((mr) => mr.state === 'opened' && mr.source_branch === source && mr.target_branch === target + && mr.source_project_id === mr.target_project_id); + if (!found) { + if (requests.data.length >= LOOKUP_PAGE) throw new Error('GitLab listed more open merge requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (hostOf(found.web_url) !== hostOf(gitlabBaseUrl())) throw new Error('GitLab answered the open merge request lookup with a URL on another host.'); + return found.web_url; +} + +function hostOf(url: string): string | undefined { + try { + return new URL(url).host; + } catch { + return undefined; + } +} diff --git a/src/providers/gitlab/index.ts b/src/providers/gitlab/index.ts index 5a603cee5..69defc2cf 100644 --- a/src/providers/gitlab/index.ts +++ b/src/providers/gitlab/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGitLabAvailable, @@ -7,6 +7,7 @@ import { gitlabRepoClone, gitlabCreateRepo, gitlabMrCreate, + gitlabFindOpenMr, GitLabRepoNotFoundError, } from './gitlab-api.js'; import { gitlabListOrgRepos } from './org.js'; @@ -22,8 +23,7 @@ import type { MRData } from '../../types.js'; */ export class GitLabProvider implements GitProvider { readonly name = 'gitlab'; - /** Workflow delivery cannot look up GitLab merge requests yet, so it opens none. */ - readonly pullRequests = null; + readonly pullRequests: PullRequestLookup = { findOpen: gitlabFindOpenMr }; parseRepoInput(input: string): RepoInfo { return parseGitLabRepoInput(input); From 3e4246cc531e5ee6c10a228070d4b488c3c9f731 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:15:01 +0200 Subject: [PATCH 19/77] feat(tgit): find the open merge request of a delivered branch Delivery looks for an open request before it opens one, so a retry or a crash between create and receipt never opens a second. TGit had no lookup, so a git.woa.com origin delivered nothing. The lookup asks the v3 REST API for opened requests from the source branch into the target and keeps one from the same project, opened or reopened. It fails rather than answering none when TGit cannot be asked or answers in time, answers what TeamAI cannot read, fills a whole page without the branch, or names a request on another host. Red: both TGit lookup tests failed with `tgit has no pull request lookup`, and the origin test for `https://git.woa.com/acme/app.git` failed with `expected false to deeply equal [ 'tgit', 'acme/app' ]`. --- src/__tests__/workflow-pr-providers.test.ts | 34 ++++++++++++++++++ src/providers/tgit/gf-cli.ts | 40 +++++++++++++++++++++ src/providers/tgit/index.ts | 6 ++-- src/providers/tgit/repo-url.ts | 2 +- 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index c8602c47a..60e51aba6 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { GitHubProvider } from '../providers/github/index.js'; import { GitLabProvider } from '../providers/gitlab/index.js'; +import { TGitProvider } from '../providers/tgit/index.js'; import { GenericGitProvider } from '../providers/git/index.js'; import { providerForRemote } from '../providers/registry.js'; import type { GitProvider, PullRequestLookup, PullRequestRef } from '../providers/types.js'; @@ -127,12 +128,45 @@ describe('GitLab merge request lookup', () => { }); }); +describe('TGit merge request lookup', () => { + const tgitMr = (source: string, iid = 7, sourceProject = 1, state = 'opened', url = `https://git.woa.com/acme/app/merge_requests/${iid}`) => ({ + ...mr(source, iid, sourceProject, url), state, + }); + beforeEach(() => vi.stubEnv('TGIT_TOKEN', 'test-token')); + + it('finds the open or reopened request of this project branch into the target, never one from a fork with the same branch name', async () => { + const fetcher = answering([tgitMr(ref.source, 6, 2), tgitMr('other', 8), tgitMr(ref.source, 9, 1, 'reopened')]); + + expect(await lookupOf(new TGitProvider()).findOpen(ref)).toBe('https://git.woa.com/acme/app/merge_requests/9'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('source_branch'), url.searchParams.get('target_branch')]) + .toEqual(['https://git.woa.com/api/v3/projects/acme%2Fapp/merge_requests', 'opened', ref.source, 'main']); + answering([tgitMr(ref.source, 6, 2), tgitMr(ref.source, 5, 1, 'merged')]); + expect(await lookupOf(new TGitProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new TGitProvider()); + answering({ message: 'private-token-in-body' }, 404); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^TGit answered the open merge request lookup with HTTP 404\.(?!.*private-token-in-body)/); + answering([{ web_url: 'https://git.woa.com/acme/app/merge_requests/7', state: 'opened' }]); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^TGit answered the open merge request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => tgitMr(`other-${index}`, index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open merge requests than one page'); + answering([tgitMr(ref.source, 7, 1, 'opened', 'https://evil.example/acme/app/merge_requests/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); +}); + describe('the provider of a checkout origin', () => { it.each([ ['https://user:secret@github.com/acme/app.git', 'github', 'acme/app'], ['ssh://git@github.com/acme/app.git', 'github', 'acme/app'], ['git@github.com:acme/app.git', 'github', 'acme/app'], ['git@gitlab.com:acme/platform/app.git', 'gitlab', 'acme/platform/app'], + ['https://git.woa.com/acme/app.git', 'tgit', 'acme/app'], ])('reads the provider and the repository from %s', (remote, provider, repo) => { vi.stubEnv('GITLAB_URL', ''); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); const origin = providerForRemote(remote); diff --git a/src/providers/tgit/gf-cli.ts b/src/providers/tgit/gf-cli.ts index 3f928b80a..822440f6e 100644 --- a/src/providers/tgit/gf-cli.ts +++ b/src/providers/tgit/gf-cli.ts @@ -8,7 +8,10 @@ import { getUserHome, legacyUserHome } from '../../utils/home.js'; import { log, spinner } from '../../utils/logger.js'; import { isInteractive } from '../../utils/prompt.js'; import { getTeamaiHomeDir } from '../../types.js'; +import { z } from 'zod'; +import type { PullRequestRef } from '../types.js'; import { tgitFetch, tgitGitCloneUrl } from './rest-auth.js'; +import { TGIT_HOST } from './repo-url.js'; /** Path where gf CLI is installed */ function gfInstallDir(): string { @@ -552,3 +555,40 @@ export function gfMrCreate(opts: GfMrCreateOptions): string { // Could not extract MR URL — treat as failure throw new Error(`gf mr create succeeded but returned unexpected output: ${output}`); } + +// ─── Open merge request lookup ─────────────────────────── + +const LOOKUP_PAGE = 100; + +const TGitOpenMrSchema = z.object({ + web_url: z.string(), + state: z.string(), + source_branch: z.string(), + target_branch: z.string(), + source_project_id: z.number(), + target_project_id: z.number(), +}); + +/** + * The open merge request from `source` into `target` in project `repo`, asked of the git.woa.com REST API with the + * credential `tgitFetch` resolves; a reopened request is open too. A request from a fork with the same branch name is + * not it. Never prompts, and throws, rather than answering none, when TGit cannot be asked or answers in time, answers + * what this cannot read, fills a whole page without it, or names a request on another host. + */ +export async function tgitFindOpenMr({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const query = new URLSearchParams({ state: 'opened', source_branch: source, target_branch: target, per_page: String(LOOKUP_PAGE) }); + const response = await tgitFetch(`/projects/${encodeURIComponent(repo)}/merge_requests?${query}`, { + redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`TGit answered the open merge request lookup with HTTP ${response.status}. Check TGIT_TOKEN or the gf login and their access to the project.`); + const requests = z.array(TGitOpenMrSchema).safeParse(await response.json().catch(() => undefined)); + if (!requests.success) throw new Error('TGit answered the open merge request lookup with data TeamAI cannot read.'); + const found = requests.data.find((mr) => (mr.state === 'opened' || mr.state === 'reopened') && mr.source_branch === source + && mr.target_branch === target && mr.source_project_id === mr.target_project_id); + if (!found) { + if (requests.data.length >= LOOKUP_PAGE) throw new Error('TGit listed more open merge requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (!found.web_url.startsWith(`https://${TGIT_HOST}/`)) throw new Error('TGit answered the open merge request lookup with a URL on another host.'); + return found.web_url; +} diff --git a/src/providers/tgit/index.ts b/src/providers/tgit/index.ts index 8ae8c58c4..3534e897e 100644 --- a/src/providers/tgit/index.ts +++ b/src/providers/tgit/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGfInstalled, @@ -8,6 +8,7 @@ import { gfRepoClone, gfCreateRepo, gfMrCreate, + tgitFindOpenMr, RepoNotFoundError as GfRepoNotFoundError, } from './gf-cli.js'; import { gfListOrgRepos } from './gf-org.js'; @@ -15,8 +16,7 @@ import { parseTGitRepoInput } from './repo-url.js'; export class TGitProvider implements GitProvider { readonly name = 'tgit'; - /** Workflow delivery cannot look up TGit merge requests yet, so it opens none. */ - readonly pullRequests = null; + readonly pullRequests: PullRequestLookup = { findOpen: tgitFindOpenMr }; parseRepoInput(input: string): RepoInfo { return parseTGitRepoInput(input); diff --git a/src/providers/tgit/repo-url.ts b/src/providers/tgit/repo-url.ts index cf87b9ff3..7a3b408ae 100644 --- a/src/providers/tgit/repo-url.ts +++ b/src/providers/tgit/repo-url.ts @@ -1,6 +1,6 @@ import type { RepoInfo } from '../types.js'; -const TGIT_HOST = 'git.woa.com'; +export const TGIT_HOST = 'git.woa.com'; /** * Parse user input into a standardized RepoInfo structure for TGit. From ff56180f339cca3599dff83315bf4c48141592b0 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:16:49 +0200 Subject: [PATCH 20/77] feat(gitcode): find the open pull request of a delivered branch Delivery looks for an open request before it opens one, so a retry or a crash between create and receipt never opens a second. GitCode had no lookup, so a gitcode.com origin delivered nothing. The lookup asks the v5 REST API for open requests from the source branch into the target and keeps one whose head and base are this repository. It fails rather than answering none when GitCode cannot be asked or answers in time, answers what TeamAI cannot read (including a state outside open, closed and merged), fills a whole page without the branch, or names a request on another host. Red: both GitCode lookup tests failed with `gitcode has no pull request lookup`, and the origin test for `https://gitcode.com/acme/app.git` failed with `expected false to deeply equal [ 'gitcode', 'acme/app' ]`. --- src/__tests__/workflow-pr-providers.test.ts | 34 ++++++++++++++++++ src/providers/gitcode/gitcode-api.ts | 40 +++++++++++++++++++++ src/providers/gitcode/index.ts | 6 ++-- 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 60e51aba6..0a8e3da3e 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { GitHubProvider } from '../providers/github/index.js'; import { GitLabProvider } from '../providers/gitlab/index.js'; import { TGitProvider } from '../providers/tgit/index.js'; +import { GitCodeProvider } from '../providers/gitcode/index.js'; import { GenericGitProvider } from '../providers/git/index.js'; import { providerForRemote } from '../providers/registry.js'; import type { GitProvider, PullRequestLookup, PullRequestRef } from '../providers/types.js'; @@ -160,6 +161,38 @@ describe('TGit merge request lookup', () => { }); }); +describe('GitCode pull request lookup', () => { + const gitcodePull = (head: string, repo = 'acme/app', number = 7, state = 'open', url = `https://gitcode.com/acme/app/pull/${number}`) => ({ + ...pull(head, repo, number), html_url: url, state, + }); + beforeEach(() => vi.stubEnv('GITCODE_TOKEN', 'test-token')); + + it('finds the open request of this repository branch into the target, never a fork with the same branch name', async () => { + const fetcher = answering([gitcodePull(ref.source, 'fork/app', 6), gitcodePull('other', 'acme/app', 8), gitcodePull(ref.source)]); + + expect(await lookupOf(new GitCodeProvider()).findOpen(ref)).toBe('https://gitcode.com/acme/app/pull/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('head'), url.searchParams.get('base')]) + .toEqual(['https://api.gitcode.com/api/v5/repos/acme/app/pulls', 'open', ref.source, 'main']); + answering([gitcodePull(ref.source, 'fork/app', 6), gitcodePull(ref.source, 'acme/app', 5, 'merged')]); + expect(await lookupOf(new GitCodeProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new GitCodeProvider()); + answering({ message: 'private-token-in-body' }, 401); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitCode answered the open pull request lookup with HTTP 401\.(?!.*private-token-in-body)/); + answering([gitcodePull(ref.source, 'acme/app', 7, 'opened')]); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitCode answered the open pull request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => gitcodePull(`other-${index}`, 'acme/app', index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open pull requests than one page'); + answering([gitcodePull(ref.source, 'acme/app', 7, 'open', 'https://evil.example/acme/app/pull/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); +}); + describe('the provider of a checkout origin', () => { it.each([ ['https://user:secret@github.com/acme/app.git', 'github', 'acme/app'], @@ -167,6 +200,7 @@ describe('the provider of a checkout origin', () => { ['git@github.com:acme/app.git', 'github', 'acme/app'], ['git@gitlab.com:acme/platform/app.git', 'gitlab', 'acme/platform/app'], ['https://git.woa.com/acme/app.git', 'tgit', 'acme/app'], + ['https://gitcode.com/acme/app.git', 'gitcode', 'acme/app'], ])('reads the provider and the repository from %s', (remote, provider, repo) => { vi.stubEnv('GITLAB_URL', ''); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); const origin = providerForRemote(remote); diff --git a/src/providers/gitcode/gitcode-api.ts b/src/providers/gitcode/gitcode-api.ts index f51c155d0..37cf7ec41 100644 --- a/src/providers/gitcode/gitcode-api.ts +++ b/src/providers/gitcode/gitcode-api.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { z } from 'zod'; +import type { PullRequestRef } from '../types.js'; import { GITCODE_HOST } from './repo-url.js'; import { getUserHome, legacyUserHome } from '../../utils/home.js'; import { sanitizeGitUrl } from '../../utils/redact.js'; @@ -311,3 +313,41 @@ export async function gitcodePullCreate(opts: GitCodePullCreateOptions): Promise if (pr.number) return `https://${GITCODE_HOST}/${opts.repo}/pull/${pr.number}`; throw new Error('GitCode PR created but response did not include html_url.'); } + +// ─── Open pull request lookup ──────────────────────────── + +const LOOKUP_PAGE = 100; + +/** A listed pull request; a state outside Gitee's three is one this cannot read. */ +const GitCodeOpenPullSchema = z.object({ + html_url: z.string(), + state: z.enum(['open', 'closed', 'merged']), + head: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }).nullable() }), + base: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }) }), +}); + +/** + * The open pull request from `source` into `target` in `repo`, asked of the GitCode REST API with the token + * `getGitCodeToken` resolves. A request from a fork with the same branch name is not it. Never prompts, and throws, + * rather than answering none, when GitCode cannot be asked or answers in time, answers what this cannot read, fills a + * whole page without it, or names a request on another host. + */ +export async function gitcodeFindOpenPull({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const token = requireToken(); + const query = new URLSearchParams({ state: 'open', head: source, base: target, per_page: String(LOOKUP_PAGE) }); + const resp = await fetch(`${gitcodeApiBase()}/repos/${repo}/pulls?${query}`, { + headers: authHeaders(token), redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!resp.ok) throw new Error(`GitCode answered the open pull request lookup with HTTP ${resp.status}. Check GITCODE_TOKEN and its access to the repository.`); + const pulls = z.array(GitCodeOpenPullSchema).safeParse(await resp.json().catch(() => undefined)); + if (!pulls.success) throw new Error('GitCode answered the open pull request lookup with data TeamAI cannot read.'); + const ours = (name: string | undefined): boolean => name?.toLowerCase() === repo.toLowerCase(); + const found = pulls.data.find((pull) => pull.state === 'open' && pull.head.ref === source && pull.base.ref === target + && ours(pull.head.repo?.full_name) && ours(pull.base.repo.full_name)); + if (!found) { + if (pulls.data.length >= LOOKUP_PAGE) throw new Error('GitCode listed more open pull requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (!found.html_url.startsWith(`https://${GITCODE_HOST}/`)) throw new Error('GitCode answered the open pull request lookup with a URL on another host.'); + return found.html_url; +} diff --git a/src/providers/gitcode/index.ts b/src/providers/gitcode/index.ts index 6370e003e..09360873e 100644 --- a/src/providers/gitcode/index.ts +++ b/src/providers/gitcode/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { gitcodeIsAuthenticated, @@ -6,6 +6,7 @@ import { gitcodeRepoClone, gitcodeCreateRepo, gitcodePullCreate, + gitcodeFindOpenPull, writeNetrcToken, GitCodeRepoNotFoundError, } from './gitcode-api.js'; @@ -26,8 +27,7 @@ import { askQuestion, isInteractive } from '../../utils/prompt.js'; */ export class GitCodeProvider implements GitProvider { readonly name = 'gitcode'; - /** Workflow delivery cannot look up GitCode pull requests yet, so it opens none. */ - readonly pullRequests = null; + readonly pullRequests: PullRequestLookup = { findOpen: gitcodeFindOpenPull }; parseRepoInput(input: string): RepoInfo { return parseGitCodeRepoInput(input); From 64971ab3853860b3b40534ac27789d95b14c01f4 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:17:31 +0200 Subject: [PATCH 21/77] test(cnb): pin that a CNB origin delivers over git alone Delivery must find an open request before it opens one, or a retry opens a second. Nothing in this code or its docs confirms how the cnb CLI lists open pull requests, and even the post-pull output it parses on create is unconfirmed, so a CNB lookup could answer a false "none". CNB keeps no lookup, the comment now says why, and a test pins that its origin is reached through git alone and opens no pull request. Red: none; the CNB origin test passed on arrival, because this commit pins existing behaviour and changes no code path. --- src/__tests__/workflow-pr-providers.test.ts | 2 +- src/providers/cnb/index.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 0a8e3da3e..a1b5c993e 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -207,7 +207,7 @@ describe('the provider of a checkout origin', () => { expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual([provider, repo]); }); - it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme'])( + it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme', 'https://cnb.cool/acme/app.git'])( 'reaches %s through git alone, which opens no pull request', (remote) => expect(providerForRemote(remote)).toEqual({ kind: 'transport' }), ); diff --git a/src/providers/cnb/index.ts b/src/providers/cnb/index.ts index 903fa52ee..52b718a66 100644 --- a/src/providers/cnb/index.ts +++ b/src/providers/cnb/index.ts @@ -20,7 +20,11 @@ import { */ export class CNBProvider implements GitProvider { readonly name = 'cnb'; - /** Workflow delivery cannot look up CNB pull requests yet, so it opens none. */ + /** + * None, so workflow delivery pushes to a CNB origin and opens no pull request. Delivery must find an open request + * before it opens one, and neither this code nor its docs confirm how the `cnb` CLI lists open pull requests (even the + * `post-pull` output `cnbPullCreate` parses is unconfirmed), so a lookup could answer a false "none" and open a second. + */ readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { From 5f8adfe6b8e0ddc5523865863aab35cbdc8202f2 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:27:53 +0200 Subject: [PATCH 22/77] fix(workflow): refuse an open pull request a lookup finds on another host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delivery now compares every URL a provider lookup answers with the host of the origin's pull requests, in the one place lookups are consumed. A URL on another host is a fault: delivery opens nothing, fails with the branch kept on origin, and records no receipt. GitHub's lookup had no such check, and a new adapter gets it without writing one. The origin hostname is already the web host for an ssh or scp remote: a self-hosted GitLab detects the remote's own host, and refuses a GITLAB_URL on another host than TEAMAI_GITLAB_HOST. A test pins that for the https, ssh and scp forms. Red: "fails, opening none, when the lookup names an open pull request on another host than the origin" failed with `expected { data: { lease: { …(6) }, …(1) } } to match object { Object (data) }`: the other host's URL was recorded and the dependent was leased. --- src/__tests__/workflow-delivery.test.ts | 16 ++++++++++++++++ src/__tests__/workflow-pr-providers.test.ts | 10 ++++++++++ src/providers/registry.ts | 6 +++++- src/workflow/pr-delivery.ts | 21 ++++++++++++--------- 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 192285759..6b1d64cef 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -294,6 +294,22 @@ describe('delivering a step', () => { expect(handed.ok && handed.value.steps[0]).toEqual(run.steps[0]); }); + it('fails, opening none, when the lookup names an open pull request on another host than the origin', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { open: [{ ...pull(7, branch), html_url: 'https://github.example.net/fixture/app/pull/7' }] }); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.deliveries).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.`) } }); + expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + }); + it('keeps the work on origin when GitHub cannot be reached, stores none of its output, and retry delivers without running the step again', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index a1b5c993e..464020092 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -207,6 +207,16 @@ describe('the provider of a checkout origin', () => { expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual([provider, repo]); }); + it.each([ + ['https://github.com/acme/app.git', '', 'github.com'], + ['ssh://git@github.com:22/acme/app.git', '', 'github.com'], + ['git@gl.example.com:acme/app.git', 'https://gl.example.com:8443', 'gl.example.com'], + ])('reads the host the pull requests of %s are on', (remote, gitlabUrl, host) => { + vi.stubEnv('GITLAB_URL', gitlabUrl); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); + const origin = providerForRemote(remote); + expect(origin.kind === 'provider' && origin.hostname).toBe(host); + }); + it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme', 'https://cnb.cool/acme/app.git'])( 'reaches %s through git alone, which opens no pull request', (remote) => expect(providerForRemote(remote)).toEqual({ kind: 'transport' }), diff --git a/src/providers/registry.ts b/src/providers/registry.ts index f2a20c7cb..c054c66fe 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -216,7 +216,11 @@ export function getProviderFromUrl(repoUrl: string): GitProvider { // ─── Workflow delivery ─────────────────────────────────── -/** A checkout's origin as workflow delivery reaches it: through a provider that can look up its pull requests, or git alone. */ +/** + * A checkout's origin as workflow delivery reaches it: through a provider that can look up its pull requests, or git + * alone. `hostname` is the host the origin's pull requests are on: its web host, which an ssh or scp remote shares, since + * a self-hosted GitLab refuses a GITLAB_URL on another host than TEAMAI_GITLAB_HOST. + */ export type OriginProvider = | { kind: 'provider'; provider: GitProvider; pullRequests: PullRequestLookup; repo: string; hostname: string } | { kind: 'transport' }; diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 75bad2ad0..8dd103233 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -5,7 +5,7 @@ */ import os from 'node:os'; import { providerForRemote, type OriginProvider } from '../providers/registry.js'; -import type { PullRequestLookup, PullRequestRef } from '../providers/types.js'; +import type { PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import type { DeliveryFailure } from './step-state.js'; import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; @@ -75,9 +75,9 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { const origin = await originOf(checkout); if (origin.kind === 'transport' || new URL(url).hostname !== origin.hostname) return undefined; - const found = await findOpen(origin.pullRequests, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); + const found = await findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); if (found.kind === 'none') return `branch ${branch} has no open pull request into ${target}`; if (found.kind === 'found' && found.pr !== url) return `is not the open pull request of branch ${branch} into ${target}`; return undefined; @@ -120,19 +120,22 @@ async function originOf(checkout: string): Promise { return remote.code === 0 ? providerForRemote(remote.stdout) : { kind: 'transport' }; } -/** What a lookup said: the open request, sanitized; none; no answer; or a URL that cannot be kept. */ +/** + * What a lookup said: the open request, sanitized; none; no answer; or a URL that cannot be kept, which a URL on another + * host than the origin's pull requests is, whichever provider answered. + */ async function findOpen( - lookup: PullRequestLookup, ref: PullRequestRef, + { pullRequests, hostname }: Extract, ref: PullRequestRef, ): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' } | { kind: 'unsafe' }> { let url: string | undefined; try { - url = await lookup.findOpen(ref); + url = await pullRequests.findOpen(ref); } catch { return { kind: 'failed' }; } if (url === undefined) return { kind: 'none' }; const clean = sanitizePrUrl(url); - return clean.url ? { kind: 'found', pr: clean.url } : { kind: 'unsafe' }; + return clean.url && new URL(clean.url).hostname === hostname ? { kind: 'found', pr: clean.url } : { kind: 'unsafe' }; } /** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ From 72685919c344efdb4f4bec20c53463ca2b323d11 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:35:44 +0200 Subject: [PATCH 23/77] fix(workflow): record the URL a host lists for a pull request it just opened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After every create, delivery looks the branch up again and records the URL the lookup gives for the open request into the same base. Only when the lookup does not list it does the receipt keep the create's answer, sanitized. A host may answer a create with another URL than its lookup: TGit's fallback builds /-/merge_requests/N. A later delivery of the same branch finds the request by lookup, so recording the create's URL listed one request twice in prs. Two tests that pinned the exact gh calls now expect the lookup after the create. Red: "lists a pull request once when its host answers the create with another URL than its lookup gives" failed with `expected [ …(2) ] to deeply equal [ Array(1) ]`: prs held the create's URL and the lookup's. --- src/__tests__/workflow-delivery.test.ts | 22 ++++++++++++++++++++-- src/workflow/pr-delivery.ts | 16 ++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 6b1d64cef..973aab6c8 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -260,7 +260,8 @@ describe('delivering a step', () => { expect(await take(config, runId)).toMatchObject({ step: 'review' }); expect((await state(config, runId)).deliveries).toEqual([{ url: 'https://github.com/fixture/app/pull/1', branch, base: 'main', step: 'fix', pass: 1 }]); - const [lookup, create, ...rest] = gh.calls(); + const [lookup, create, again, ...rest] = gh.calls(); + expect(again?.args).toEqual(lookup?.args); expect(rest).toEqual([]); expect(lookup?.args[0]).toBe('api'); expect(lookup?.prompt).toBe('0'); @@ -573,7 +574,24 @@ describe('where a pull request goes', () => { expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); expect(creates(gh)).toEqual([[`teamai/${runId}/change/p1`, 'main']]); - expect(gh.calls().filter((call) => call.args[0] === 'api')).toHaveLength(3); + // The first pass looks the branch up before and after it opens the request, and each later pass finds it. + expect(gh.calls().filter((call) => call.args[0] === 'api')).toHaveLength(4); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('lists a pull request once when its host answers the create with another URL than its lookup gives', async () => { + const { config, cwd, runId } = await fixture( + ` - id: change\n workspace: new\n workflow:\n steps:\n - id: fix\n workspace: continue\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 2", max_iterations: 2 }\n`, + ); + const branch = `teamai/${runId}/change/p1`; + const open: string[] = []; + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + open.push('https://github.com/fixture/app/pull/1'); + return 'https://github.com/fixture/app/-/merge_requests/1'; + }); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(create).toHaveBeenCalledTimes(1); expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 8dd103233..a71885359 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -64,7 +64,7 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) /** * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch - * is looked up, and only when there is none, and `gate` agrees, is one opened; a create that fails is looked up again. + * is looked up, and only when there is none, and `gate` agrees, is one opened; then it is looked up again. * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { @@ -82,20 +82,24 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise | undefined; try { - const created = sanitizePrUrl(await provider.createPullRequest({ + created = sanitizePrUrl(await provider.createPullRequest({ repo, source: job.key.branch, target: job.target, title: `${job.runName}: ${job.key.step}`, description: `Opened by TeamAI for review: workflow ${job.runName}, run ${job.runId}, step ${job.key.step}, pass ${job.key.pass}.\n\nBase commit: ${job.base}\n\nTeamAI never merges this pull request.`, // Run outside the member checkout, so the provider's CLI can neither push from it nor read its branch. cwd: os.tmpdir(), spawnTimeoutMs: DELIVERY_TIMEOUT_MS, })); - return created.url ? { ok: true, pr: created.url } : failed({ kind: 'unsafe_url', provider: provider.name }); } catch { - // The host may have opened the request before the create failed or timed out: find it rather than open another. - const again = await findOpen(origin, ref); - return again.kind === 'found' ? { ok: true, pr: again.pr } : failed({ kind: 'create_failed', provider: provider.name }); + created = undefined; } + // A create that failed or timed out may still have opened the request. One that answered may name it by another URL + // than the lookup a later delivery of the branch finds it by, so the lookup's is kept: one request, one receipt. + const opened = await findOpen(origin, ref); + if (opened.kind === 'found') return { ok: true, pr: opened.pr }; + if (!created) return failed({ kind: 'create_failed', provider: provider.name }); + return created.url ? { ok: true, pr: created.url } : failed({ kind: 'unsafe_url', provider: provider.name }); } /** From 7a799be65c5f840133cf5781392a37aa52ecc5b7 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 13:51:33 +0200 Subject: [PATCH 24/77] test(workflow): run the delivery scenarios against each kind of driver through the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit conformance.ts gains the delivery scenarios every driver that takes workspace leases must pass, next to the git scenarios: a delivered branch opens one pull request into the run base before the step that needs it runs; a process killed inside the create is followed by a next that finds the request and opens no other; a run published while it delivers and claimed by a second install with its own checkout opens one; a cancel during the lookup opens none; a cancel during the create lists it while the step stays cancelled; an unreachable host fails naming the branch, which stays on origin; and a driver polls while another process of its machine delivers. The existing scenario format cannot kill a process or hold a provider call, so workflow-delivery-drivers.test.ts runs these scenarios through the built CLI, by name, for four drivers: the local driver, a session driver that commits in its own worktree and pushes, an external driver that knows nothing of delivery and sleeps on every wait, and a remote driver that opens its own pull request from another clone and reports it. The fake gh every case shares can now hold a lookup or a create until the runner has acted. Red: with the gate before the create always open, "local|session| external: a run cancelled while its delivery looks for an open pull request opens none" failed with `expected [ [ 'fixture/app', …(2) ] ] to deeply equal []`: the cancelled run opened a pull request. --- .../e2e/workflow-delivery-drivers.test.ts | 427 ++++++++++++++++++ src/__tests__/helpers/fake-gh.ts | 30 +- src/workflow/conformance.ts | 93 ++++ 3 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 src/__tests__/e2e/workflow-delivery-drivers.test.ts diff --git a/src/__tests__/e2e/workflow-delivery-drivers.test.ts b/src/__tests__/e2e/workflow-delivery-drivers.test.ts new file mode 100644 index 000000000..87462e795 --- /dev/null +++ b/src/__tests__/e2e/workflow-delivery-drivers.test.ts @@ -0,0 +1,427 @@ +/** + * Real CLI: the delivery scenarios of `conformance.ts` for each kind of driver, through `teamai workflow` JSON alone. Two + * installs, each with its own HOME and checkout, share a team origin and a product origin whose URL is + * `https://github.com/fixture/app.git`, which git rewrites to a bare repository. One fake `gh` on PATH is the host of + * every case: it records each create, answers lookups from what it recorded, and holds the call a case names. + * + * - local: `workflow start --driver local` runs the run; after a crash, `workflow resume` continues it. + * - session: commits in a worktree of the install's checkout, pushes the branch itself, reports, then calls next. + * - external: knows nothing of delivery. It works in a clone of its own and sleeps on every `wait`, whatever it says. + * - remote: works in a clone on another machine, opens its own pull request there, and reports it as changes.pr_url. + */ +import { execFileSync, execSync, spawn, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { deliveryConformanceWorkflow, workflowDeliveryConformanceScenarios, type DeliveryConformanceScenario } from '../../workflow/conformance.js'; +import { fakeGhHost, type FakeGh } from '../helpers/fake-gh.js'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const CLI = path.join(ROOT, 'dist', 'index.js'); +const TEAM_URL = 'https://github.com/acme/team.git'; +const PRODUCT_URL = 'https://github.com/fixture/app.git'; +const POLL_MS = 250; +const HOLD_MS = 2_000; + +type Profile = 'local' | 'session' | 'external' | 'remote'; +type Member = 'alice' | 'bob'; + +const EnvelopeSchema = z.object({ + data: z.record(z.unknown()).optional(), + error: z.object({ code: z.string(), message: z.string() }).passthrough().optional(), +}); +type Envelope = z.infer; + +const LeaseSchema = z.object({ + step: z.string(), attempt: z.string(), revision: z.number(), command: z.string().optional(), + workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }), start: z.object({ commit: z.string() }) }).optional(), +}); +type Lease = z.infer; + +const ReceiptSchema = z.object({ url: z.string(), branch: z.string(), base: z.string(), step: z.string(), pass: z.number() }); +const RunFileSchema = z.object({ + status: z.string(), + steps: z.array(z.object({ + id: z.string(), status: z.string(), + changes: z.object({ branch: z.string(), to: z.string() }).optional(), + failure: z.object({ code: z.string(), message: z.string() }).optional(), + })), + deliveries: z.array(ReceiptSchema).optional(), +}); +type RunFile = z.infer; +const RecordSchema = z.object({ run: z.string(), prs: z.array(z.string()).optional() }); +const OpenSchema = z.array(z.object({ html_url: z.string(), head: z.object({ ref: z.string() }) })); + +interface Result { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +let sandbox = ''; +let teamOrigin = ''; +let productOrigin = ''; +let gh: FakeGh; +let ghDir = ''; +let clones = 0; + +const homeOf = (member: Member): string => path.join(sandbox, member); +const workOf = (member: Member): string => path.join(homeOf(member), 'work'); +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +function envOf(member: Member): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: homeOf(member), USERPROFILE: homeOf(member), GIT_CONFIG_GLOBAL: path.join(homeOf(member), '.gitconfig'), + GH_CONFIG_DIR: path.join(homeOf(member), '.gh'), PATH: `${ghDir}${path.delimiter}${process.env.PATH ?? ''}`, + GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', FORCE_COLOR: '0', + SHIP_OPEN: path.join(ghDir, 'open.json'), SHIP_SAW: path.join(ghDir, 'ship-saw.json'), + }; + for (const name of ['XDG_CONFIG_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'GITHUB_TOKEN', 'GH_TOKEN']) delete env[name]; + return env; +} + +function git(cwd: string, args: string[], env: NodeJS.ProcessEnv = process.env): string { + return execFileSync('git', args, { cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +} + +function teamai(member: Member, args: string[], options: { cwd?: string; spawned?: (child: ChildProcess) => void; stderr?: (text: string) => void } = {}): Promise { + return new Promise((resolve) => { + const child = spawn(process.execPath, [CLI, ...args], { cwd: options.cwd ?? workOf(member), env: envOf(member), stdio: ['ignore', 'pipe', 'pipe'] }); + options.spawned?.(child); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString(); options.stderr?.(stderr); }); + child.on('close', (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +function envelope(result: Result): Envelope { + const line = result.stdout.trim().split('\n').filter((item) => item.startsWith('{')).at(-1); + if (!line) throw new Error(`no JSON envelope:\n${said(result)}`); + return EnvelopeSchema.parse(JSON.parse(line)); +} + +function said(result: Result): string { + return `exit ${result.code ?? result.signal}\n${result.stdout}\n${result.stderr}`; +} + +async function waitFor(check: () => boolean, ms: number): Promise { + for (const until = Date.now() + ms; !check(); await sleep(50)) if (Date.now() > until) return false; + return true; +} + +function seed(dir: string, files: Record, origin: string): void { + fs.mkdirSync(dir, { recursive: true }); + git(dir, ['init', '-q', '-b', 'main']); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), text); + } + git(dir, ['add', '-A']); + git(dir, ['-c', 'user.name=Seed', '-c', 'user.email=seed@example.com', 'commit', '-qm', 'seed']); + git(dir, ['push', '-q', origin, 'main']); +} + +async function join(member: Member): Promise { + fs.mkdirSync(path.join(homeOf(member), '.gh'), { recursive: true }); + fs.writeFileSync(path.join(homeOf(member), '.gitconfig'), [ + '[user]', `\tname = ${member}`, `\temail = ${member}@example.com`, + `[url "file://${teamOrigin}"]`, `\tinsteadOf = ${TEAM_URL}`, + `[url "file://${productOrigin}"]`, `\tinsteadOf = ${PRODUCT_URL}`, '', + ].join('\n')); + git(sandbox, ['clone', '-q', PRODUCT_URL, workOf(member)], envOf(member)); + for (const args of [['init', TEAM_URL, '--provider', 'git', '--agent', 'claude', '--scope', 'user', '--force'], ['pull']]) { + const result = await teamai(member, args); + expect(result.code, said(result)).toBe(0); + } +} + +function readRun(member: Member, run: string): RunFile { + return RunFileSchema.parse(JSON.parse(fs.readFileSync(path.join(homeOf(member), '.teamai', 'workflows', 'runs', `${run}.json`), 'utf8'))); +} + +function endRecordPrs(run: string): string[] | undefined { + const text = git(sandbox, ['--git-dir', teamOrigin, 'show', `teamai-reports:workflows/runs/${run}.json`]); + return RecordSchema.parse(JSON.parse(text)).prs; +} + +function createsOf(run: string): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr' && call.args[1] === 'create' && call.args.includes(`teamai/${run}/work/p1`)) + .map((call) => ['-R', '-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +/** What a stub driver did: the steps it was leased and reported, its next calls, those killed, and the waits it slept on. */ +interface DriverLog { + leases: string[]; + reports: string[]; + nexts: number; + killed: number; + waits: number; +} + +/** + * A driver that speaks only `teamai workflow next|report` JSON. It runs each leased command, in the workspace its + * profile keeps for a lease with one, commits and pushes that branch itself, and reports. It sleeps on every `wait`, and + * calls `next` again when one was killed, unless `stopped` says it was stopped. + */ +async function stubDriver(profile: Exclude, member: Member, run: string, options: { + claim?: boolean; + afterReport?: (step: string) => Promise; + stopped?: () => boolean; + reportedUrl?: (branch: string) => { open: string; report: string }; + onReport?: (step: string, result: Result) => void; +} = {}): Promise { + const log: DriverLog = { leases: [], reports: [], nexts: 0, killed: 0, waits: 0 }; + const driver = { kind: profile, machine: `${member}-${profile}`, workspace: workOf(member), can: { workspace: ['new', 'continue', 'read-only'] } }; + let claim = options.claim === true; + for (let calls = 0; calls < 80; calls++) { + if (options.stopped?.()) return log; + const next = await teamai(member, ['workflow', 'next', run, '--json', JSON.stringify({ driver, ...(claim ? { claim: true } : {}) })]); + log.nexts++; + if (next.signal) { + log.killed++; + continue; + } + claim = false; + const answer = envelope(next); + if (answer.error) throw new Error(`next answered an error:\n${said(next)}`); + if (answer.data?.done) return log; + if (answer.data?.wait) { + log.waits++; + await sleep(POLL_MS); + continue; + } + const lease = LeaseSchema.parse(answer.data?.lease); + log.leases.push(lease.step); + const reported = await work(profile, member, lease, options.reportedUrl); + const body = { attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver, ...reported }; + const result = await teamai(member, ['workflow', 'report', run, '--json', JSON.stringify(body)]); + log.reports.push(lease.step); + options.onReport?.(lease.step, result); + expect(result.code === 0 || envelope(result).error?.code === 'lease_cancelled', said(result)).toBe(true); + await options.afterReport?.(lease.step); + } + throw new Error(`${profile} driver did not finish run ${run}: ${JSON.stringify(log)}`); +} + +/** Runs a lease's command where `profile` keeps its workspace, and returns the `changes` the report carries. */ +async function work(profile: Exclude, member: Member, lease: Lease, reportedUrl?: (branch: string) => { open: string; report: string }): Promise> { + const env = envOf(member); + if (!lease.workspace) { + execSync(lease.command ?? 'true', { cwd: workOf(member), env, stdio: 'ignore' }); + return {}; + } + const { branch, base, start } = lease.workspace; + const dir = path.join(sandbox, `${member}-${profile}-${clones++}`); + if (profile === 'session') git(workOf(member), ['worktree', 'add', '-q', '-b', branch, dir, start.commit], env); + else { + git(sandbox, ['clone', '-q', PRODUCT_URL, dir], env); + git(dir, ['checkout', '-q', '-b', branch, start.commit], env); + } + execSync(lease.command ?? 'true', { cwd: dir, env, stdio: 'ignore' }); + git(dir, ['add', '-A'], env); + git(dir, ['commit', '-qm', lease.step], env); + git(dir, ['push', '-q', 'origin', branch], env); + const to = git(dir, ['rev-parse', 'HEAD'], env); + const changes: Record = { branch, base: base.commit, from: start.commit, to, commits: [to] }; + if (profile !== 'remote') return { changes }; + const url = (reportedUrl ?? ((head) => ({ open: openOwn(head), report: '' })))(branch); + return { changes: { ...changes, pr_url: url.report || `${url.open}?tab=files#top` } }; +} + +/** Opens a pull request from `head` into main on the host, as a remote driver's own tooling does, and returns its URL. */ +function openOwn(head: string): string { + const file = path.join(ghDir, 'open.json'); + const open = fs.existsSync(file) ? z.array(z.unknown()).parse(JSON.parse(fs.readFileSync(file, 'utf8'))) : []; + const url = `https://github.com/fixture/app/pull/${900 + open.length}`; + const repo = { full_name: 'fixture/app' }; + fs.writeFileSync(file, JSON.stringify([...open, { state: 'open', html_url: url, head: { ref: head, repo }, base: { ref: 'main', repo } }])); + return url; +} + +/** Starts the run on alice's install, drives it with `profile`, and does what the scenario's hold says while it holds. */ +async function drive(profile: Profile, scenario: DeliveryConformanceScenario): Promise<{ run: string; member: Member; log?: DriverLog; held: boolean; killed: boolean }> { + for (const marker of ['release', 'lookup.held', 'create.held', 'ship-saw.json']) fs.rmSync(path.join(ghDir, marker), { force: true }); + gh.configure({ ...(scenario.unreachable ? { lookup: 'fail' as const } : {}), ...(scenario.hold ? { hold: scenario.hold.call } : {}) }); + const heldFile = scenario.hold && path.join(ghDir, `${scenario.hold.call}.held`); + const release = (): void => fs.writeFileSync(path.join(ghDir, 'release'), ''); + let run = ''; + let stopped = false; + let finished = false; + let watcher: Promise | undefined; + + let driving: Promise; + if (profile === 'local') { + driving = teamai('alice', ['workflow', 'start', 'deliver', '--driver', 'local'], { stderr: (text) => { + run ||= /Run (r[0-9a-f]{16}) started/.exec(text)?.[1] ?? ''; + } }); + expect(await waitFor(() => run !== '', 30_000), 'the local driver printed its run').toBe(true); + } else { + const started = await teamai('alice', ['workflow', 'start', 'deliver']); + run = String(envelope(started).data?.run); + driving = stubDriver(profile, 'alice', run, { + stopped: () => stopped, + // A process of the same machine that polls without taking leases starts the delivery and holds it. + afterReport: async (step) => { + if (step !== 'work' || scenario.hold?.action !== 'poll' || !heldFile) return; + const poller = { kind: 'watcher', machine: 'alice', workspace: workOf('alice') }; + let exited = false; + watcher = teamai('alice', ['workflow', 'next', run, '--json', JSON.stringify({ driver: poller, canTakeLease: false })]); + void watcher.then(() => { exited = true; }); + await waitFor(() => fs.existsSync(heldFile) || exited, 30_000); + }, + }); + } + void driving.then(() => { finished = true; }, () => { finished = true; }); + + const held = heldFile !== undefined && await waitFor(() => fs.existsSync(heldFile) || finished, 60_000) && fs.existsSync(heldFile); + let member: Member = 'alice'; + let log: DriverLog | undefined; + let killed = false; + if (held && scenario.hold && heldFile) { + const pid = Number(fs.readFileSync(heldFile, 'utf8')); + switch (scenario.hold.action) { + case 'crash': + process.kill(pid, 'SIGKILL'); + release(); + break; + case 'publish': { + const published = await teamai('alice', ['workflow', 'publish', run]); + expect(envelope(published), said(published)).toMatchObject({ data: { publication: 'published' } }); + stopped = true; + process.kill(pid, 'SIGKILL'); + release(); + await driving.catch(() => undefined); + member = 'bob'; + break; + } + case 'cancel': { + const cancelled = await teamai('alice', ['workflow', 'cancel', run]); + expect(cancelled.code, said(cancelled)).toBe(0); + release(); + break; + } + case 'poll': + await sleep(HOLD_MS); + release(); + break; + default: { + const unknown: never = scenario.hold.action; + throw new Error(`unknown hold action ${String(unknown)}`); + } + } + } + if (member === 'bob') { + if (profile === 'local') { + const resumed = await teamai('bob', ['workflow', 'resume', run]); + expect(envelope(resumed), said(resumed)).toMatchObject({ data: { done: { run } } }); + } else { + log = await stubDriver(profile, 'bob', run, { claim: true }); + } + } else { + const driven = await driving; + if ('leases' in driven) { + log = driven; + killed = driven.killed > 0; + } else if (driven.signal) { + killed = true; + const resumed = await teamai('alice', ['workflow', 'resume', run]); + expect(envelope(resumed), said(resumed)).toMatchObject({ data: { done: { run } } }); + } + } + if (watcher) await watcher; + return { run, member, log, held, killed }; +} + +const PROFILES: Profile[] = ['local', 'session', 'external', 'remote']; + +describe('workflow delivery conformance of each kind of driver through the CLI', () => { + beforeAll(async () => { + sandbox = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-delivery-drivers-'))); + teamOrigin = path.join(sandbox, 'team.git'); + productOrigin = path.join(sandbox, 'app.git'); + ghDir = path.join(sandbox, 'gh'); + git(sandbox, ['init', '-q', '--bare', '-b', 'main', teamOrigin]); + git(sandbox, ['init', '-q', '--bare', '-b', 'main', productOrigin]); + seed(path.join(sandbox, 'seed-team'), { 'teamai.yaml': `team: acme\nrepo: ${TEAM_URL}\nprovider: github\n`, 'workflows/deliver.yaml': deliveryConformanceWorkflow }, teamOrigin); + seed(path.join(sandbox, 'seed-app'), { 'README.md': '# app\n' }, productOrigin); + gh = fakeGhHost(ghDir); + await join('alice'); + await join('bob'); + }, 120_000); + + afterAll(() => { + if (sandbox) fs.rmSync(sandbox, { recursive: true, force: true }); + }); + + for (const scenario of workflowDeliveryConformanceScenarios) { + for (const profile of PROFILES) { + // A local driver calls next only in its own process, so no other process can start its delivery first. + const runs = !(profile === 'local' && scenario.hold?.action === 'poll'); + it.skipIf(!runs)(`${profile}: ${scenario.name}`, async () => { + const expected = profile === 'remote' ? scenario.reported : scenario.opened; + const { run, member, log, held, killed } = await drive(profile, scenario); + const state = readRun(member, run); + const branch = `teamai/${run}/work/p1`; + const work = state.steps.find((step) => step.id === 'work'); + const steps = state.steps.map((step) => `${step.id} ${step.status}${step.failure ? `: ${step.failure.message}` : ''}`).join('\n'); + + // A driver that reports its own pull request is looked up only when it reports, and never reaches a create. + const holds = profile === 'remote' ? scenario.hold?.call === 'lookup' : scenario.hold !== undefined; + expect(held, 'the case reached the call it holds').toBe(holds); + expect(killed, 'the process inside the create was killed').toBe(holds && scenario.hold?.action === 'crash'); + if (holds && scenario.hold?.action === 'poll') expect(log?.waits, 'the driver polled while the delivery held').toBeGreaterThan(0); + expect([state.status, work?.status], steps).toEqual([expected.run, expected.work]); + expect(createsOf(run)).toEqual(Array.from({ length: expected.creates }, () => ['fixture/app', branch, 'main'])); + expect(state.deliveries ?? []).toHaveLength(expected.prs); + expect(endRecordPrs(run) ?? []).toEqual((state.deliveries ?? []).map((receipt) => receipt.url)); + for (const receipt of state.deliveries ?? []) { + expect(receipt).toMatchObject({ branch, base: 'main', step: 'work', pass: 1 }); + expect(receipt.url).toMatch(/^https:\/\/github\.com\/fixture\/app\/pull\/\d+$/); + } + if (expected.failure) expect(work?.failure).toMatchObject({ code: expected.failure, message: expect.stringContaining(branch) }); + const tip = git(sandbox, ['--git-dir', productOrigin, 'rev-parse', `refs/heads/${branch}`]); + if (work?.changes) expect(tip, 'the branch is on origin').toBe(work.changes.to); + if (expected.run === 'succeeded') { + const saw = OpenSchema.parse(JSON.parse(fs.readFileSync(path.join(ghDir, 'ship-saw.json'), 'utf8'))); + expect(saw.filter((pull) => pull.head.ref === branch).map((pull) => pull.html_url), 'ship ran once work was delivered') + .toEqual((state.deliveries ?? []).map((receipt) => receipt.url)); + } + if (profile === 'external' && log) { + expect(log.reports, 'it reports only what it was leased, and never settles a delivering step').toEqual(log.leases); + expect(log.waits, 'it sleeps on each wait').toBeLessThanOrEqual(HOLD_MS / POLL_MS + 2); + expect(log.nexts, 'its next calls stay bounded').toBeLessThanOrEqual(log.leases.length + log.waits + 2); + } + }, 120_000); + } + } + + it('remote: a reported URL with credentials is named in the report envelope without its value, and the open pull request of the branch is delivered', async () => { + gh.configure({}); + const started = await teamai('alice', ['workflow', 'start', 'deliver']); + const run = String(envelope(started).data?.run); + let opened = ''; + const reports: Result[] = []; + await stubDriver('remote', 'alice', run, { + reportedUrl: (branch) => { + opened = openOwn(branch); + return { open: opened, report: opened.replace('https://', 'https://x-access-token:ghs_droppedvalue@') }; + }, + onReport: (step, result) => { if (step === 'work') reports.push(result); }, + }); + const [report] = reports; + expect(report && envelope(report)).toMatchObject({ data: { accepted: true, step: { id: 'work', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); + expect(report && said(report)).not.toContain('ghs_droppedvalue'); + const state = readRun('alice', run); + expect(state.status).toBe('succeeded'); + expect(state.deliveries?.map((receipt) => receipt.url)).toEqual([opened]); + expect(createsOf(run)).toEqual([]); + expect(JSON.stringify(state)).not.toContain('ghs_droppedvalue'); + }, 120_000); +}); diff --git a/src/__tests__/helpers/fake-gh.ts b/src/__tests__/helpers/fake-gh.ts index 110ed27d5..c4431846c 100644 --- a/src/__tests__/helpers/fake-gh.ts +++ b/src/__tests__/helpers/fake-gh.ts @@ -10,12 +10,15 @@ export type GhCall = z.infer; /** * How the fake answers. `lookup: 'fail'` fails every `gh api`. `create: 'fail'` fails `gh pr create` and opens nothing; * `'fail-after-open'` opens the request and still fails, as a create that timed out after GitHub took it. `open` - * replaces the open requests. + * replaces the open requests. `hold` waits inside that call, a create once it recorded the request, until a file named + * `release` is in the fake's directory, or 20 seconds passed; while it waits, `.held` there holds the pid of the + * process that called it. */ export interface FakeGhOptions { lookup?: 'fail'; create?: 'fail' | 'fail-after-open'; open?: unknown[]; + hold?: 'lookup' | 'create'; } function script(dir: string): string { @@ -28,8 +31,14 @@ const args = process.argv.slice(2); const options = read('options.json', {}); fs.appendFileSync(path.join(dir, 'calls.jsonl'), JSON.stringify({ args, cwd: process.cwd(), prompt: process.env.GIT_TERMINAL_PROMPT }) + '\\n'); const open = read('open.json', []); +const hold = (call) => { + if (options.hold !== call) return; + fs.writeFileSync(path.join(dir, call + '.held'), String(process.ppid)); + for (const until = Date.now() + 20000; !fs.existsSync(path.join(dir, 'release')) && Date.now() < until;) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); +}; if (args[0] === 'api') { if (options.lookup === 'fail') { console.error('HTTP 401: Bad credentials for token ghp_fake_secret'); process.exit(1); } + hold('lookup'); console.log(JSON.stringify(open)); process.exit(0); } @@ -38,6 +47,7 @@ if (args[0] === 'pr' && args[1] === 'create') { const repo = value('-R'); const pull = { state: 'open', html_url: 'https://github.com/' + repo + '/pull/' + (open.length + 1), head: { ref: value('-H'), repo: { full_name: repo } }, base: { ref: value('-B'), repo: { full_name: repo } } }; if (options.create !== 'fail') fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify([...open, pull])); + hold('create'); if (options.create) { console.error('pull request create failed for token ghp_fake_secret'); process.exit(1); } console.log(pull.html_url); process.exit(0); @@ -47,16 +57,31 @@ process.exit(9); `; } +export interface FakeGh { + /** The executable, named `gh`. */ + bin: string; + configure(next: FakeGhOptions): void; + calls(): GhCall[]; +} + /** * A `gh` that keeps open pull requests under `dir` and logs each call with its cwd and GIT_TERMINAL_PROMPT. It answers * `gh api` with every open request and `gh pr create` with a new one, and its failures print a token, so a test can * show none is kept. teamai resolves `gh` to it until the test's mocks are restored. */ -export function fakeGh(dir: string, options: FakeGhOptions = {}): { configure(next: FakeGhOptions): void; calls(): GhCall[] } { +export function fakeGh(dir: string, options: FakeGhOptions = {}): FakeGh { + const fake = fakeGhHost(dir, options); + resolveGh(fake.bin); + return fake; +} + +/** The same `gh` for a process this test spawns, which finds it first on its PATH. */ +export function fakeGhHost(dir: string, options: FakeGhOptions = {}): FakeGh { fs.mkdirSync(dir, { recursive: true }); const bin = path.join(dir, 'gh'); fs.writeFileSync(bin, script(dir), { mode: 0o755 }); const fake = { + bin, configure(next: FakeGhOptions): void { fs.writeFileSync(path.join(dir, 'options.json'), JSON.stringify(next)); if (next.open) fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify(next.open)); @@ -68,7 +93,6 @@ export function fakeGh(dir: string, options: FakeGhOptions = {}): { configure(ne }, }; fake.configure(options); - resolveGh(bin); return fake; } diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index 34c8d91c9..ee258be5e 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -905,6 +905,99 @@ export const workflowGitConformanceScenarios: GitConformanceScenario[] = [ }, ]; +/** What a run holds once a delivery case ends, over every install that drove it. */ +export interface DeliveryExpectation { + /** Pull requests TeamAI asked the host to open. */ + creates: number; + run: 'succeeded' | 'failed' | 'cancelled'; + work: 'succeeded' | 'failed' | 'cancelled'; + /** The pull requests the run lists in `deliveries` and its end record's `prs`. */ + prs: number; + /** The failure code of `work`, whose message then names its branch. */ + failure?: string; +} + +/** + * A delivery case for every driver that takes workspace leases, run against `deliveryConformanceWorkflow` with a fake + * provider every case shares. The runner gives the product an `https://github.com//` origin that git + * rewrites to a bare repository, and a fake `gh` that records each create and answers lookups from what it recorded. + * With `hold`, the fake waits inside that call, a create once it recorded the request, until the runner has done + * `action`: `crash` kills the process the call came from, and the driver calls `next` again; `publish` publishes the + * run and kills that process, and a driver of a second install with its own checkout claims the run and drives it; + * `cancel` cancels the run; `poll` holds a delivery that another process of the machine started after `work` reported, + * while the driver under test calls `next`. A driver whose delivery never reaches `hold` runs to the end untouched. + * `unreachable` fails every lookup. `opened` is expected of a driver that leaves the pull request to TeamAI, `reported` + * of one that reports its own as `changes.pr_url`. After every case, `work`'s branch is on origin at its reported commit. + */ +export interface DeliveryConformanceScenario { + name: string; + hold?: { call: 'lookup' | 'create'; action: 'crash' | 'publish' | 'cancel' | 'poll' }; + unreachable?: true; + opened: DeliveryExpectation; + reported: DeliveryExpectation; +} + +/** `ship` copies what the host holds when it runs to `SHIP_SAW`, which the runner sets: the open requests by then. */ +export const deliveryConformanceWorkflow = `schemaVersion: v1alpha1 +name: deliver +description: A step delivers its branch as a pull request, and the step that needs it runs once it is delivered. +steps: + - id: work + workspace: new + command: node -e "require('fs').writeFileSync('work.txt', 'W')" + deliver: pr + - id: ship + needs: [work] + command: node -e "require('fs').copyFileSync(process.env.SHIP_OPEN, process.env.SHIP_SAW)" +`; + +const DELIVERED: DeliveryExpectation = { creates: 1, run: 'succeeded', work: 'succeeded', prs: 1 }; +const REPORTED: DeliveryExpectation = { creates: 0, run: 'succeeded', work: 'succeeded', prs: 1 }; + +export const workflowDeliveryConformanceScenarios: DeliveryConformanceScenario[] = [ + { + name: 'a delivered branch opens one pull request into the run base, and the step that needs it runs once it is delivered', + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a delivery whose process stops inside the create finds the pull request on the next call and opens no other', + hold: { call: 'create', action: 'crash' }, + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a run published while it delivers, and claimed by another install with its own checkout, opens one pull request', + hold: { call: 'create', action: 'publish' }, + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a run cancelled while its delivery looks for an open pull request opens none', + hold: { call: 'lookup', action: 'cancel' }, + opened: { creates: 0, run: 'cancelled', work: 'cancelled', prs: 0 }, + reported: { creates: 0, run: 'cancelled', work: 'cancelled', prs: 0 }, + }, + { + name: 'a run cancelled while its delivery opens the pull request lists it, and the step stays cancelled', + hold: { call: 'create', action: 'cancel' }, + opened: { creates: 1, run: 'cancelled', work: 'cancelled', prs: 1 }, + reported: REPORTED, + }, + { + name: 'a delivery that cannot reach the host fails naming the branch, which stays on origin', + unreachable: true, + opened: { creates: 0, run: 'failed', work: 'failed', prs: 0, failure: 'delivery_failed' }, + reported: REPORTED, + }, + { + name: 'a driver polls while another process of its machine delivers, and gets the step that needs it once delivered', + hold: { call: 'create', action: 'poll' }, + opened: DELIVERED, + reported: REPORTED, + }, +]; + export const judgeLeaseConformanceScenario = { name: 'a judge lease renews through running and settles only through score submission', yaml: `schemaVersion: v1alpha1 From 2ed026a9f8038db86c0731c9edc5db100d9423ae Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 15:49:49 +0200 Subject: [PATCH 25/77] test(workflow): run the local driver while another process delivers The delivery scenario where a driver polls while another process of its machine delivers skipped the local driver, on the grounds that only its own process calls next. workflow resume can drive a run whose step a session driver reported and another process delivers, and the local driver must poll there rather than exit busy. The cell now runs that way. The external driver's cells also check that its report leaves the step delivering, so the step settles through TeamAI, not through the driver. Red: with the local driver's poll on wait.delivering removed, "local: a driver polls while another process of its machine delivers, and gets the step that needs it once delivered" failed with `the driver kept driving while the delivery held: expected false to be true`: the local driver exited busy before the delivery was released. --- .../e2e/workflow-delivery-drivers.test.ts | 73 ++++++++++++------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/src/__tests__/e2e/workflow-delivery-drivers.test.ts b/src/__tests__/e2e/workflow-delivery-drivers.test.ts index 87462e795..b810f6254 100644 --- a/src/__tests__/e2e/workflow-delivery-drivers.test.ts +++ b/src/__tests__/e2e/workflow-delivery-drivers.test.ts @@ -4,7 +4,8 @@ * `https://github.com/fixture/app.git`, which git rewrites to a bare repository. One fake `gh` on PATH is the host of * every case: it records each create, answers lookups from what it recorded, and holds the call a case names. * - * - local: `workflow start --driver local` runs the run; after a crash, `workflow resume` continues it. + * - local: `workflow start --driver local` runs the run; after a crash, `workflow resume` continues it. While another + * process delivers, `workflow resume` drives a run another driver reported. * - session: commits in a worktree of the install's checkout, pushes the branch itself, reports, then calls next. * - external: knows nothing of delivery. It works in a clone of its own and sleeps on every `wait`, whatever it says. * - remote: works in a clone on another machine, opens its own pull request there, and reports it as changes.pr_url. @@ -156,10 +157,15 @@ function createsOf(run: string): string[][] { .map((call) => ['-R', '-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); } -/** What a stub driver did: the steps it was leased and reported, its next calls, those killed, and the waits it slept on. */ +const ReportAnswerSchema = z.object({ step: z.object({ status: z.string() }) }); + +/** + * What a stub driver did: the steps it was leased, the steps it reported with the status each report answered, its + * next calls, those killed, and the waits it slept on. + */ interface DriverLog { leases: string[]; - reports: string[]; + reports: Array<{ step: string; status: string }>; nexts: number; killed: number; waits: number; @@ -202,9 +208,10 @@ async function stubDriver(profile: Exclude, member: Member, ru const reported = await work(profile, member, lease, options.reportedUrl); const body = { attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver, ...reported }; const result = await teamai(member, ['workflow', 'report', run, '--json', JSON.stringify(body)]); - log.reports.push(lease.step); + const answered = envelope(result); + log.reports.push({ step: lease.step, status: ReportAnswerSchema.safeParse(answered.data).data?.step.status ?? answered.error?.code ?? '' }); options.onReport?.(lease.step, result); - expect(result.code === 0 || envelope(result).error?.code === 'lease_cancelled', said(result)).toBe(true); + expect(result.code === 0 || answered.error?.code === 'lease_cancelled', said(result)).toBe(true); await options.afterReport?.(lease.step); } throw new Error(`${profile} driver did not finish run ${run}: ${JSON.stringify(log)}`); @@ -245,8 +252,11 @@ function openOwn(head: string): string { return url; } -/** Starts the run on alice's install, drives it with `profile`, and does what the scenario's hold says while it holds. */ -async function drive(profile: Profile, scenario: DeliveryConformanceScenario): Promise<{ run: string; member: Member; log?: DriverLog; held: boolean; killed: boolean }> { +/** + * Starts the run on alice's install, drives it with `profile`, and does what the scenario's hold says while it holds. + * `polled` says the driver was still driving when a `poll` hold was released. + */ +async function drive(profile: Profile, scenario: DeliveryConformanceScenario): Promise<{ run: string; member: Member; log?: DriverLog; held: boolean; killed: boolean; polled: boolean }> { for (const marker of ['release', 'lookup.held', 'create.held', 'ship-saw.json']) fs.rmSync(path.join(ghDir, marker), { force: true }); gh.configure({ ...(scenario.unreachable ? { lookup: 'fail' as const } : {}), ...(scenario.hold ? { hold: scenario.hold.call } : {}) }); const heldFile = scenario.hold && path.join(ghDir, `${scenario.hold.call}.held`); @@ -254,10 +264,26 @@ async function drive(profile: Profile, scenario: DeliveryConformanceScenario): P let run = ''; let stopped = false; let finished = false; + let polled = false; let watcher: Promise | undefined; + // A process of the same machine that polls without taking leases starts the delivery and holds it. + const deliverElsewhere = async (step: string): Promise => { + if (step !== 'work' || scenario.hold?.action !== 'poll' || !heldFile) return; + const poller = { kind: 'watcher', machine: 'alice', workspace: workOf('alice') }; + let exited = false; + watcher = teamai('alice', ['workflow', 'next', run, '--json', JSON.stringify({ driver: poller, canTakeLease: false })]); + void watcher.then(() => { exited = true; }); + await waitFor(() => fs.existsSync(heldFile) || exited, 30_000); + }; let driving: Promise; - if (profile === 'local') { + if (profile === 'local' && scenario.hold?.action === 'poll') { + // The local driver delivers in its own process right after it reports, so here a session driver reports work and + // stops while another process delivers it, and the local driver resumes the run meanwhile. + run = String(envelope(await teamai('alice', ['workflow', 'start', 'deliver'])).data?.run); + await stubDriver('session', 'alice', run, { afterReport: deliverElsewhere, stopped: () => watcher !== undefined }); + driving = teamai('alice', ['workflow', 'resume', run]); + } else if (profile === 'local') { driving = teamai('alice', ['workflow', 'start', 'deliver', '--driver', 'local'], { stderr: (text) => { run ||= /Run (r[0-9a-f]{16}) started/.exec(text)?.[1] ?? ''; } }); @@ -265,18 +291,7 @@ async function drive(profile: Profile, scenario: DeliveryConformanceScenario): P } else { const started = await teamai('alice', ['workflow', 'start', 'deliver']); run = String(envelope(started).data?.run); - driving = stubDriver(profile, 'alice', run, { - stopped: () => stopped, - // A process of the same machine that polls without taking leases starts the delivery and holds it. - afterReport: async (step) => { - if (step !== 'work' || scenario.hold?.action !== 'poll' || !heldFile) return; - const poller = { kind: 'watcher', machine: 'alice', workspace: workOf('alice') }; - let exited = false; - watcher = teamai('alice', ['workflow', 'next', run, '--json', JSON.stringify({ driver: poller, canTakeLease: false })]); - void watcher.then(() => { exited = true; }); - await waitFor(() => fs.existsSync(heldFile) || exited, 30_000); - }, - }); + driving = stubDriver(profile, 'alice', run, { stopped: () => stopped, afterReport: deliverElsewhere }); } void driving.then(() => { finished = true; }, () => { finished = true; }); @@ -309,6 +324,7 @@ async function drive(profile: Profile, scenario: DeliveryConformanceScenario): P } case 'poll': await sleep(HOLD_MS); + polled = !finished; release(); break; default: { @@ -336,7 +352,7 @@ async function drive(profile: Profile, scenario: DeliveryConformanceScenario): P } } if (watcher) await watcher; - return { run, member, log, held, killed }; + return { run, member, log, held, killed, polled }; } const PROFILES: Profile[] = ['local', 'session', 'external', 'remote']; @@ -362,11 +378,9 @@ describe('workflow delivery conformance of each kind of driver through the CLI', for (const scenario of workflowDeliveryConformanceScenarios) { for (const profile of PROFILES) { - // A local driver calls next only in its own process, so no other process can start its delivery first. - const runs = !(profile === 'local' && scenario.hold?.action === 'poll'); - it.skipIf(!runs)(`${profile}: ${scenario.name}`, async () => { + it(`${profile}: ${scenario.name}`, async () => { const expected = profile === 'remote' ? scenario.reported : scenario.opened; - const { run, member, log, held, killed } = await drive(profile, scenario); + const { run, member, log, held, killed, polled } = await drive(profile, scenario); const state = readRun(member, run); const branch = `teamai/${run}/work/p1`; const work = state.steps.find((step) => step.id === 'work'); @@ -376,7 +390,10 @@ describe('workflow delivery conformance of each kind of driver through the CLI', const holds = profile === 'remote' ? scenario.hold?.call === 'lookup' : scenario.hold !== undefined; expect(held, 'the case reached the call it holds').toBe(holds); expect(killed, 'the process inside the create was killed').toBe(holds && scenario.hold?.action === 'crash'); - if (holds && scenario.hold?.action === 'poll') expect(log?.waits, 'the driver polled while the delivery held').toBeGreaterThan(0); + if (holds && scenario.hold?.action === 'poll') { + expect(polled, 'the driver kept driving while the delivery held').toBe(true); + if (log) expect(log.waits, 'the driver polled while the delivery held').toBeGreaterThan(0); + } expect([state.status, work?.status], steps).toEqual([expected.run, expected.work]); expect(createsOf(run)).toEqual(Array.from({ length: expected.creates }, () => ['fixture/app', branch, 'main'])); expect(state.deliveries ?? []).toHaveLength(expected.prs); @@ -394,7 +411,9 @@ describe('workflow delivery conformance of each kind of driver through the CLI', .toEqual((state.deliveries ?? []).map((receipt) => receipt.url)); } if (profile === 'external' && log) { - expect(log.reports, 'it reports only what it was leased, and never settles a delivering step').toEqual(log.leases); + expect(log.reports.map((report) => report.step), 'it reports only what it was leased').toEqual(log.leases); + expect(log.reports.filter((report) => report.step === 'work').map((report) => report.status), 'its report leaves work delivering, for TeamAI to settle') + .toEqual(log.leases.includes('work') ? ['delivering'] : []); expect(log.waits, 'it sleeps on each wait').toBeLessThanOrEqual(HOLD_MS / POLL_MS + 2); expect(log.nexts, 'its next calls stay bounded').toBeLessThanOrEqual(log.leases.length + log.waits + 2); } From b46f190ec98f7da1d6906bd41cf506dd2b4bf6c8 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 16:05:00 +0200 Subject: [PATCH 26/77] docs(workflow): document pull request delivery and run the CI recipe as written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both workflow guides now say where a pull request goes, when TeamAI opens or reuses one, what fails a delivery and how retry continues it, how a driver reports its own, and what prs in a run record holds, with the jq query that finds the run behind a pull request. Statements that no longer hold are gone, and the CI recipe's start line picks the prompt tool with --tools, since a runner has no local tools config and a prompt step without one fails tool_unconfigured. The workflow engine notes now name the switches a new step status must fit, how next delivers outside the run lock, and the readers and writers of run.deliveries, copies[].from and delivery.creating. The cancel help and guide list delivering steps, and the retry help and guide say a failed delivery is retried in its own pass. workflow-pr-delivery-1090.test.ts now reads the recipe's job from the guide and runs its commands with bash against a fixture product and a fake gh: a GitHub origin gets one pull request into the checked-out branch, listed in the record; an origin reached through git alone fails naming the branch on origin; a required approval fails hitl_required. The docs test checks that every recipe job has runs-on and that both languages show the same commands. Red: "finds the run that opened the pull request with the jq query of the guide" failed with `docs/guide/workflow.md shows no jq query over workflows/runs`. With runs-on removed from the Chinese recipe, the runs-on check failed with `job ship has no runs-on`; with --tools removed there, "shows the same CI recipe commands in every language" failed with `expected [ [ …(3) ] ] to deeply equal [ [ …(3) ] ]`. --- docs/dev/workflow-engine.md | 14 +- docs/guide/workflow-advanced.md | 4 +- docs/guide/workflow.md | 32 +- docs/guide/zh-CN/workflow-advanced.md | 4 +- docs/guide/zh-CN/workflow.md | 32 +- skill-data/core/references/commands.md | 4 +- .../e2e/workflow-pr-delivery-1090.test.ts | 319 +++++++++++++----- src/__tests__/helpers/ci-recipe.ts | 26 ++ src/__tests__/workflow-docs-examples.test.ts | 14 + src/index.ts | 4 +- 10 files changed, 330 insertions(+), 123 deletions(-) create mode 100644 src/__tests__/helpers/ci-recipe.ts diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 7a000292f..a258fb250 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -11,8 +11,10 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. - **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. -- **A workspace is a git contract on the lease; the driver executes it, and PR delivery inspects its committed result.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. +- **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreation`), runs `deliverPr` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery`, which reads the immutable `DeliveryJob` rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL or creates, looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and sets `delivery.creating`. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `completeDelivery` writes. `deliveryTarget` in `workspace.ts` reads which branches delivered, `unresolvedCreation` reads whether a step's pass has a receipt, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `delivery.creating` is set by `markCreating` and stays when the step settles or is cancelled, since `cancel` keeps `delivery`. On a step that no longer delivers, in an ended run too, `unresolvedCreation` reads it and, with no receipt for that pass, only looks the request up. `creationResolved` clears it once a lookup answered found or none. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. @@ -24,7 +26,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **An expression reads its absolute paths, wherever they are.** `expressionPaths` returns every `inputs` and `steps` path, including those inside function arguments and predicates, and never a relative path such as `.size`, which reads the current element of a predicate. `readsOf`, `skipUnreachable` and the pull read checks use it, so a new kind of expression must return its paths there. - **Expression types are checked in two places, through `typeProblems`.** `parseExpression` rejects what an expression shows by itself, such as `x < 'a'` or `any(...) < 1`. `readDiagnostics` checks paths against the types the workflow declares. It checks each path only in the workflow that owns it, so a predicate an inline step runs over an enclosing step is reported once. - **Pull and the engine share one list of templated fields, one list of step expressions, and one list of actions.** The templated fields are in `templatedTexts` and the expressions, such as `if` and `break`, in `stepExpressions`, both in `template.ts`. HITL declarations normalize in `parse-hitl.ts`. The actions are in `STEP_ACTIONS` in `types.ts`. Add a new field or action there, not to a copy in `parse.ts` or `engine.ts`. Pull, the cache's `prompt_file` check, the lease render and `readsOf` all split a template with `parseTemplate`, which decides which `{{ … }}` is a template, and the render evaluates it with `evaluateExpression`, as for an `if`. `parseReads` in `engine.ts` and the step parse in `parse.ts` key a table by each expression, so a new one does not compile until both say how to read it. -- **A new step status must fit `stopped()`, `settled()` and `stopWhere()`.** The first two are in `step-state.ts`, and `stopWhere()` is in `engine.ts`. `stopped()` decides which dependents are skipped, which steps can recover, and the run outcome. `settled()` decides readiness. `stopWhere()` decides what an end or a break does to each step under it. Each switches over every status, so a status added to `StepStateSchema` does not compile until all three place it. +- **A new step status must fit five switches and two predicates.** `stopped()`, `settled()` and `takeOver()` are in `step-state.ts`, `stopWhere()` in `engine.ts`, and `retryInstance()` in `run-controls.ts`. `stopped()` decides which dependents are skipped, which steps can recover, and the run outcome. `settled()` decides readiness. `takeOver()` decides what a claim does to each step, `stopWhere()` what an end or a break does to each step under it, and `retryInstance()` what a retry does to each step in its cone. The predicates `inProgress()` and `cancellable()` in `step-state.ts` decide whether `settleScope` waits on a step and whether `cancelRun` may stop it. Each of the seven switches over every status with a `never` default, so a status added to `StepStateSchema` does not compile until all of them place it. `StepStateSchema` and the snapshot's `StepCheckpointSchema` are built from one variant list, `SHARED_VARIANTS`, so the status also reaches a handed-over run; they differ only in the `leased` variant. - **Only the transitions in `step-state.ts` write step fields.** `waitForAnswer`, `answerHitl`, `resetPending`, `resetIteration`, `skip`, `lease`, `release`, `start`, `succeed`, `fail`, `cancel`, `repend`, `nextRound`, `endLoop` and `exhaust` change a step in place and drop the fields its new status does not have. `readRun` parses every step through `StepStateSchema`, which gives a run file from an older teamai its defaults: `settledAt` 0, `outcome` from `status`, and no fields its status does not have. - **`run.steps` is replaced, never changed in place.** Its type is `readonly`, so `addItems`, `insertChildren` and `archiveUnder` add or remove instances by assigning a new array. An instance's `id` is `readonly` too, so an instance cannot move to another scope's group. `statesIn` in `scope.ts` keeps the instances of every scope per array, in a `WeakMap`, so settling a scope reads its own instances without scanning every step. A new array starts a new index, except that `insertChildren`, which adds the instances of a workflow a step starts, carries the old index over. - **A pull check that changes what a file means carries a `code`.** `dropsKeptVersion` in `cache.ts` removes the cached copy when a diagnostic has a `code`. An older definition then cannot keep running with the new meaning. A diagnostic without a `code` keeps the last valid version, but only after `checkReferences` checks that version again against what the workflows it runs settle to in the same sync. @@ -42,10 +44,4 @@ Each rule below spans more than one file. The rule names the code that owns it, ## Tests -PR delivery (`delivery.ts`) runs after the agent's successful report or composite settlement, in a durable `delivering` state. It is neither stopped nor settled, counts toward fan-out concurrency, and can be cancelled. A provider failure becomes `delivery_failed`; retry preserves the branch and returns to delivery, without another agent attempt. `next` reconciles first, then drains deliveries outside the run lock under a separate reclaimable delivery lock; provider lookup makes a restart after creation idempotent. Drivers poll a transient `wait.delivery`. No delivery code calls a merge API. - -Shared delivery state readers/writers: `rememberCopy` records `copies[].origin` once from the actual workspace start/first merge, before the driver updates `copies[].commit` in `report`. Workspace planning reads commit/owner; delivery walks origin to the nearest validated delivered branch, falling back to `run.base.ref`. `changes.pr_url` is validated outside the lock and accepted only by the report transaction that owns the lease. The engine stamps `prRecordedAt`; drivers cannot set it. Step schemas, loop passes and archives retain both fields. End records read only these changes fields, order by revision, sanitize again and deduplicate; output URLs are excluded and reporting opt-out applies unchanged. `done` gains no fields. - -Delivery inspects the exact reported commit against the run base using `git diff --no-renames`, with shared `AGENT_PATHS` plus CI/hook paths. It never trusts a reported diff or commit list. All providers use their existing authentication, with optional open-request lookup and repository/branch validation; arbitrary HTTPS review services remain allowed without sending them provider credentials. Generic Git retains transport-only behavior and reports the preserved branch on failure. - -Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass the protocol scenarios in `src/workflow/conformance.ts`, and a driver that takes workspace leases the git scenarios there too. If a driver can see a new behavior, add a scenario there. +Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass the protocol scenarios in `src/workflow/conformance.ts`, and a driver that takes workspace leases the git and delivery scenarios there too. `workflow-delivery-drivers.test.ts` runs the delivery scenarios through the built CLI for the local driver, a driver that pushes from its own worktree, one that knows nothing of delivery, and one that reports its own pull request. If a driver can see a new behavior, add a scenario there. diff --git a/docs/guide/workflow-advanced.md b/docs/guide/workflow-advanced.md index ee2785a10..7050a1df2 100644 --- a/docs/guide/workflow-advanced.md +++ b/docs/guide/workflow-advanced.md @@ -642,11 +642,11 @@ teamai workflow cancel [step] teamai workflow resume --tools default=claude ``` -`retry` accepts a leased, failed, or cancelled step, a refused approve with an answer, or an answered choose. Retry a HITL request to change its answer; its old answer stays in pass history. It frees an old lease and starts a new pass at a new revision. The local driver stops a command it was running and starts the new pass only after that command has exited. It resets the steps that depend on or read that step, including loop waits and item `after` dependencies. A HITL request among them that waits, or was answered and has not run, is asked again at a new pass. It reopens settled enclosing workflows, items, fan-out steps, and the run. If the reopened run is still running, the result includes a `resume` command. Run it to continue locally, or let your driver poll `next`. Unanswered skips in each affected container are re-derived; manual cancellations outside it stay cancelled. An independent standing end or break prevents retry. +`retry` accepts a leased, failed, or cancelled step, a refused approve with an answer, or an answered choose. Retry a HITL request to change its answer; its old answer stays in pass history. It frees an old lease and starts a new pass at a new revision. The local driver stops a command it was running and starts the new pass only after that command has exited. It resets the steps that depend on or read that step, including loop waits and item `after` dependencies. A HITL request among them that waits, or was answered and has not run, is asked again at a new pass. It reopens settled enclosing workflows, items, fan-out steps, and the run. A step whose pull request delivery failed keeps its pass, and only its delivery runs again. If the reopened run is still running, the result includes a `resume` command. Run it to continue locally, or let your driver poll `next`. Unanswered skips in each affected container are re-derived; manual cancellations outside it stay cancelled. An independent standing end or break prevents retry. History stays in `passes` and the archive, so `length(steps.x.passes)` counts failed attempts too. Retrying a loop owner resets its iteration count. Retrying another step in its range keeps the owner's count. Retry keeps the top-level workflow's round count, so retry in its exhausted final round can fail `loop_exhausted` again. -A single-step `cancel` stops pending, waiting, leased, or running work. Its open descendants stop too. Cancelled steps count as stopped, so dependents skip unless an `if` reads their outcome for recovery. The run normally fails. A whole-run cancel ends immediately with run outcome `cancelled`, without starting another loop round. Cancel does not bump the revision. A late report returns `lease_cancelled`; `next.cancelled` tells drivers which tools to stop. +A single-step `cancel` stops pending, waiting, leased, running, or delivering work. Its open descendants stop too. Cancelled steps count as stopped, so dependents skip unless an `if` reads their outcome for recovery. The run normally fails. A whole-run cancel ends immediately with run outcome `cancelled`, without starting another loop round. Cancel does not bump the revision. A late report returns `lease_cancelled`; `next.cancelled` tells drivers which tools to stop. `resume` drives the run locally after a retry, response, Ctrl-C, or closed terminal. `workflow start --driver local` prints the run id on stderr when the run starts, with the `resume` command that continues it. It uses this machine's open local lease workspace first, then the workspace of the latest driver that called `next` when that driver was local on this machine, then the current directory. Resuming after a retry from a subdirectory therefore keeps the original local workspace. Only one local driver can drive a run at a time. A second resume returns `driver_running` with the owning process id; stop that driver or wait. Locks from dead processes are reclaimed automatically. A settled run returns `done`. Failed and cancelled local runs exit 1; successful control commands exit 0 even when their answer describes a failed or cancelled run. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index fb8c0016e..0f968c082 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -218,7 +218,7 @@ steps: ## Deliver a PR -Set `deliver: pr` on a writable step, or on a `workflow` step whose children share its `workspace: new` branch. TeamAI pushes the branch and opens a review request through the product repository's Git provider. It never merges it. A `for_each` delivers one PR per item. A looped `new` step gets a new `/p` branch and PR each pass; earlier PRs stay open. +Set `deliver: pr` on a step that writes a branch, and TeamAI opens a pull request from that branch once the step succeeds. It never merges it. A step writes a branch when it has `workspace: new` or `continue`, or when it is a prompt inside a workflow that opened one. A command that delivers must set `workspace`, since a command without one is read-only. A `workflow` step with `workspace: new` delivers its branch once, for all its steps. On a `for_each` step, each item delivers its own branch. ```yaml schemaVersion: v1alpha1 @@ -232,19 +232,23 @@ steps: prompt: "Implement issue #{{ inputs.issue }}." ``` -The PR targets the nearest delivered branch along the branch's recorded origin; `for_each.after` comes before other `needs`, and the first child inherits the enclosing workflow's dependencies. Skipped or read-only dependencies are passed over. If no earlier branch delivered, the target is the run's original branch. Start from a named branch: a detached checkout has no PR target. An open PR for the same head and target is reused. +A `deliver: pr` step that would run read-only fails validation. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. -Commands that deliver must explicitly choose `workspace: new` or `continue`: omitting workspace makes a command read-only even inside a writable workflow. Prompts can inherit an enclosing writable workspace. +When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. A run that a crash stopped, or that another machine took over, finds the pull request it opened before and opens no other. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. -Agent completion is followed by a durable `delivering` state. Dependents wait for it. Git and provider operations happen outside the run lock. A PR creation failure reports `delivery_failed` and names the preserved branch. Fix provider authentication, run `teamai workflow retry `, then `teamai workflow resume `: this retries delivery on that branch without rerunning the agent. Push failures remain `push_failed` and retain the worktree. A completed composite may already have removed its worktree; its committed, pushed branch remains. Generic Git hosts support transport only: create a PR yourself and report its URL, or use a provider that supports PR creation. No interactive login starts during delivery. +A pull request goes into the nearest branch its own branch grew from that already delivered one, or else into the branch the run started from. A branch grows first from its `for_each` item's `after` branches, then from its `needs` in order. The first step of a sub-workflow grows from the needs of the step that runs it. A skipped step wrote no branch, so it is passed over. Each pull request then shows only the work done since the branch it goes into. -Before opening a PR, TeamAI checks the delivered commit against the run's base commit. Changes to CI, hooks, agent instructions, rules, MCP and equivalent tool configuration are rejected, with the paths named. The shared agent configuration paths are the same ones restored before workspace execution. This check includes changes inherited from earlier branches and deletions/renames. +Each pass of a looped `workspace: new` step has its own `/p` branch, so it opens its own pull request. The earlier ones stay open, by design. A step that loops on one `continue` branch has one pull request. -A remote driver may include `changes.pr_url` alongside `branch`, `base`, `from`, `to` and `commits`. Recognized provider metadata is checked against the expected repository and branches. Any HTTPS host/path is allowed for private review services. URLs containing credentials, invalid/non-HTTPS URLs, and URLs longer than 512 characters are dropped; query strings and fragments are stripped. A dropped URL does not reject the report: `dropped` lists `{ field: "changes.pr_url", reason: "…" }` without the value. Unknown nested fields are listed in `ignored` as `changes.`. Drivers must still obey a lease's workspace and commit contract. +Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. + +Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. + +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep, or one from a step without `deliver: pr`, is dropped. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI -Store the workflow above in the team repo and configure a local prompt tool there. A normal GitHub Actions job is the v1 trigger path. The token must read the team repo, push branches to the product repo, and create PRs there. Configure `TEAM_REPO` and `WORKFLOW_TOKEN` in the repository settings. +A GitHub Actions job can start a run with the local driver, and the run delivers its pull requests unattended. Store the workflow in the team repo. Give the job a token that can read the team repo, push branches to the product repo, and open pull requests there. The job below reads it from a `WORKFLOW_TOKEN` secret, and the team repo's URL from a `TEAM_REPO` variable. The runner also needs the tool that runs the prompt steps, signed in, and `--tools` picks it for this start. ```yaml on: { issues: { types: [labeled] } } @@ -260,10 +264,10 @@ jobs: with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } - run: npx teamai-cli init "$TEAM_REPO" --scope workspace - run: npx teamai-cli pull - - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' + - run: npx teamai-cli workflow start ship-issue --driver local --tools default=claude --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' ``` -`--auto-approve` answers optional approve requests and optional choose defaults only. A required gate fails with `hitl_required` in an unattended job. To start from a failing branch, check that branch out by name; the default `pull_request` checkout is a detached merge commit. TeamAI does not install triggers or wait for CI/merges. +`--auto-approve` answers an optional `approve` request yes and an optional `choose` request with its default. Nobody can answer a required one in an unattended job, so its step fails with `hitl_required`, and so does the run. To start a run from another branch, such as one whose CI failed, check that branch out by name, for example `ref: ${{ github.head_ref }}` on a `pull_request` event. The default `pull_request` checkout is a detached merge commit, which `workflow start` refuses. TeamAI installs no trigger and does not wait for CI or a merge. ## Continue a run on another machine @@ -286,14 +290,18 @@ A step with `session: continue` or `session.from` starts a new agent session on ## Run records -The end record includes `prs`, the validated `changes.pr_url` links in first-appearance order, once per URL across items and loop passes. URLs from step outputs are never published. `prs` follows the same usage-report opt-out as the rest of the record and is public when the team repo is public. Older records without `prs` remain readable. - With [usage reporting](reporting.md) on, every run publishes a privacy-preserving record to the team's `teamai-reports` branch: once when it starts and once when it ends. With it off, nothing is published and the envelopes carry no `record`. Records live in `workflows/runs/.json`, keyed by run id, so a retried publication updates the same record instead of adding a copy. -A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. +A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, the pull requests the run delivered (`prs`), and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. An entry's `agent` is what that pass ran as, as observed. It holds the step's `agent:` definition and how the tool got it (`main` when the tool reads its own agent file, `prompt` when the instructions are prepended), the tool and its version, the model and effort the launch asked for and the model the tool reported, the team commit the checkout last pulled and whether a delivered file was edited since, the project commit, the TeamAI version, and the driver kind. A value nobody observed is left out. `fingerprint` hashes the definition, tool, model, effort and team commit, so two passes with the same fingerprint ran with the same recorded parts. It is left out when the model or the team commit is unknown, or a delivered file was edited. A pass nothing observed, and a record from an older TeamAI, carry only the definition's name. While a step runs, the tool it starts, and any session that tool starts in the same checkout, skips the session-start pull when the checkout's last pull delivered completely to that tool, so the step keeps the team commit it records. +`prs` lists every pull request the run delivered, whether TeamAI opened it or a driver reported it, once each, in the order the run first delivered them. A URL from a step's output is never listed. Like the rest of the record, `prs` follows the usage-report opt-out, and it is public when the team repo is public. A record from before `prs` still reads. To find the run that opened a pull request, run this in a checkout of the team repo's `teamai-reports` branch: + +```text +jq -r 'select((.prs // []) | index("")) | .run' workflows/runs/*.json +``` + The operation that settles the run queues its end record under the run lock and publishes it after releasing that lock. Retry updates the same record path when the run ends again. Answer entries publish choice, channel, and agent only; notes, answer timestamps, and answer driver identities stay local. A publication that fails — no remote, a busy reports lock, a rejected push — is queued in this scope's data directory and retried on the next `teamai pull`. The `start` and `next` envelopes then carry `record: pending` instead of `record: published`. The run itself is unaffected: a machine that dies after starting a run leaves it visible as started but not ended. Run ids are `r` plus 16 hex digits. Runs created before this scheme used 8 hex digits and still resolve. diff --git a/docs/guide/zh-CN/workflow-advanced.md b/docs/guide/zh-CN/workflow-advanced.md index 1a6690570..e7ba6a71d 100644 --- a/docs/guide/zh-CN/workflow-advanced.md +++ b/docs/guide/zh-CN/workflow-advanced.md @@ -642,11 +642,11 @@ teamai workflow cancel [step] teamai workflow resume --tools default=claude ``` -`retry` 接受 leased、failed 或 cancelled 步骤，也接受已回答并拒绝的 approve 和已回答的 choose。重试 HITL 请求可以更改答案，旧答案保留在 pass 历史中。它释放旧租约，以新修订号启动新 pass（本地驱动会停止正在运行的命令，并等它退出后才启动新 pass），重置依赖或读取该步骤的后续步骤，包括循环等待和 item 的 after 依赖（其中正在等待、或已回答但尚未运行的 HITL 请求会在新 pass 中重新询问），并重新打开已结算的外层工作流、item、fan-out 步骤和运行。如果重新打开后运行仍在进行，返回结果会包含 `resume` 命令。执行该命令继续本地运行，或让驱动轮询 `next`。每个受影响容器中未回答的跳过步骤会重新计算；范围外手动取消的步骤保持取消。无关步骤已到达的有效结局或 break 会阻止重试。 +`retry` 接受 leased、failed 或 cancelled 步骤，也接受已回答并拒绝的 approve 和已回答的 choose。重试 HITL 请求可以更改答案，旧答案保留在 pass 历史中。它释放旧租约，以新修订号启动新 pass（本地驱动会停止正在运行的命令，并等它退出后才启动新 pass），重置依赖或读取该步骤的后续步骤，包括循环等待和 item 的 after 依赖（其中正在等待、或已回答但尚未运行的 HITL 请求会在新 pass 中重新询问），并重新打开已结算的外层工作流、item、fan-out 步骤和运行。交付 pull request 失败的步骤保留原来的 pass，只重新运行交付。如果重新打开后运行仍在进行，返回结果会包含 `resume` 命令。执行该命令继续本地运行，或让驱动轮询 `next`。每个受影响容器中未回答的跳过步骤会重新计算；范围外手动取消的步骤保持取消。无关步骤已到达的有效结局或 break 会阻止重试。 历史保留在 passes 和 archive 中，因此 `length(steps.x.passes)` 也统计失败尝试。重试循环所有者会重置其迭代数；重试范围内的其他步骤则保留所有者的计数。重试保留顶层工作流的 round，耗尽最后一轮后重试仍可能再次 `loop_exhausted`。 -指定步骤的 cancel 停止 pending、waiting、leased 或 running 工作，同时停止尚未结束的后代。cancelled 属于 stopped，因此依赖步骤会跳过，除非其 if 读取取消结果来恢复；运行通常失败。取消整个运行立即以 `cancelled` 结束，不启动下一轮。cancel 不增加修订号。迟到的报告返回 `lease_cancelled`，`next.cancelled` 告诉驱动停止哪些工具。 +指定步骤的 cancel 停止 pending、waiting、leased、running 或 delivering 工作，同时停止尚未结束的后代。cancelled 属于 stopped，因此依赖步骤会跳过，除非其 if 读取取消结果来恢复；运行通常失败。取消整个运行立即以 `cancelled` 结束，不启动下一轮。cancel 不增加修订号。迟到的报告返回 `lease_cancelled`，`next.cancelled` 告诉驱动停止哪些工具。 在重试、回答、Ctrl-C 或关闭终端后，用 resume 继续本地运行。`workflow start --driver local` 在运行开始时就把运行 id 和继续它的 `resume` 命令输出到 stderr。它优先使用本机未结本地租约的工作目录，其次使用最近调用 `next` 的驱动工作目录（该驱动必须是本机本地驱动），最后使用当前目录。因此重试后即使从子目录继续，也会保留原来的本地工作目录。每个运行同时只能有一个本地驱动。重复 resume 返回 driver_running 和持有者进程号，请停止该驱动或等待；死进程留下的锁会自动回收。已结算运行返回 done。本地运行以 failed 或 cancelled 结束时退出码为 1；控制命令成功时退出码为 0，即使返回的运行状态为 failed 或 cancelled。 diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 4f1341814..352556d6c 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -218,7 +218,7 @@ steps: ## 交付 PR -在可写步骤上设置 `deliver: pr`，或在子步骤共用其 `workspace: new` 分支的 `workflow` 步骤上设置它。TeamAI 推送分支，通过产品仓库的 Git provider 创建供人审查的 PR，从不合并。`for_each` 每项交付一个 PR。循环中的 `new` 步骤每个 pass 使用新的 `/p` 分支和 PR，之前的 PR 保持打开。 +在写分支的步骤上设置 `deliver: pr`，步骤成功后 TeamAI 会从这条分支开一个 pull request，并且从不合并它。设置了 `workspace: new` 或 `continue` 的步骤会写分支，位于已开分支的工作流中的提示词步骤也会。交付的命令必须设置 `workspace`，因为没有 `workspace` 的命令是只读的。设置了 `workspace: new` 的 `workflow` 步骤为它的全部步骤只交付一次自己的分支。`for_each` 步骤的每个 item 各自交付自己的分支。 ```yaml schemaVersion: v1alpha1 @@ -232,19 +232,23 @@ steps: prompt: "Implement issue #{{ inputs.issue }}." ``` -PR 的目标沿分支记录的来源，追溯到最近已经交付的分支。`for_each.after` 优先于其他 `needs`，第一个子步骤继承外层工作流的依赖；跳过或只读的依赖会继续向前追溯。之前没有交付分支时，目标为运行开始时的分支。因此请从有名称的分支开始；detached checkout 没有 PR 目标。同一源分支和目标分支已有打开的 PR 时会复用。 +会以只读方式运行的 `deliver: pr` 步骤无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 -交付 PR 的命令必须显式选择 `workspace: new` 或 `continue`；即使位于可写工作流中，省略 workspace 的命令仍为只读。提示词可以继承外层的可写工作区。 +步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。因崩溃而中断、或被另一台机器接管的运行，会找到之前开的 pull request，不会再开第二个。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 -agent 完成后，步骤进入持久的 `delivering` 状态，后续依赖等待交付结束。Git 和 provider 操作不占用运行锁。创建 PR 失败时以 `delivery_failed` 结束，消息写明保留的分支。修好认证后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，只重试这条分支的交付，不重跑 agent。推送失败仍为 `push_failed`，并保留 worktree。组合工作流结束时可能已移除 worktree，但提交和远程分支仍在。通用 Git 主机只支持传输：请自行创建 PR 并上报 URL，或使用支持创建 PR 的 provider。交付不会启动交互登录。 +pull request 合入这条分支来源中最近一条已经交付过的分支，没有时合入运行开始时的分支。分支首先来自其 `for_each` item 的 `after` 分支，然后依次来自 `needs`。子工作流的第一个步骤来自运行它的步骤的 needs。跳过的步骤没有写分支，会被越过。因此每个 pull request 只显示它所合入的分支之后的工作。 -创建 PR 前，TeamAI 将交付提交与运行的基准提交比较。CI、hooks、agent 指令、rules、MCP 和同类工具配置的修改会被拒绝，并列出路径。共享 agent 配置路径与工作区执行前恢复的路径相同。检查也覆盖从前序分支继承的修改，以及删除和重命名。 +循环中 `workspace: new` 步骤的每个 pass 都有自己的 `/p` 分支，因此各开一个 pull request。之前的 pull request 保持打开，这是有意为之。在同一条 `continue` 分支上循环的步骤只有一个 pull request。 -远程驱动可在 `changes` 的 `branch`、`base`、`from`、`to`、`commits` 之外上报 `pr_url`。provider 能提供元数据时，会校验仓库和源、目标分支。私有审查服务可使用任意 HTTPS 主机和路径。带凭据、无效、非 HTTPS 或超过 512 字符的 URL 被丢弃；query 和 fragment 会移除。丢弃 URL 不会拒绝 report，`dropped` 以 `{ field: "changes.pr_url", reason: "…" }` 列出原因，不包含值。未知嵌套字段以 `changes.<字段>` 列入 `ignored`。驱动仍须遵守租约的工作区与提交契约。 +开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。 + +其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 + +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL，以及来自未设置 `deliver: pr` 的步骤的 URL，会被丢弃。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 -将上面的工作流放进团队仓库，并在其中配置本地提示词工具。普通 GitHub Actions 作业是 v1 的触发方式。token 需要能读取团队仓库、向产品仓库推送分支和创建 PR。在仓库设置里配置 `TEAM_REPO` 和 `WORKFLOW_TOKEN`。 +GitHub Actions 作业可以用本地驱动启动运行，运行会在无人值守的情况下交付它的 pull request。把工作流放进团队仓库。给作业一个 token，它要能读取团队仓库、向产品仓库推送分支，并在产品仓库开 pull request。下面的作业从 `WORKFLOW_TOKEN` secret 读取它，从 `TEAM_REPO` 变量读取团队仓库的 URL。runner 还需要运行提示词步骤的工具，并且已经登录，`--tools` 为这次启动选择它。 ```yaml on: { issues: { types: [labeled] } } @@ -260,10 +264,10 @@ jobs: with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } - run: npx teamai-cli init "$TEAM_REPO" --scope workspace - run: npx teamai-cli pull - - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' + - run: npx teamai-cli workflow start ship-issue --driver local --tools default=claude --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' ``` -`--auto-approve` 只回答可选 approve 和有默认值的可选 choose。必需的 gate 在无人值守作业中以 `hitl_required` 失败。从失败分支启动时，按分支名检出；默认的 `pull_request` checkout 是 detached merge commit。TeamAI 不安装触发器，也不等待 CI 或合并事件。 +`--auto-approve` 对可选的 `approve` 请求回答 yes，对可选的 `choose` 请求选择默认值。无人值守的作业里没人能回答必需的请求，因此它所在的步骤以 `hitl_required` 失败，运行也随之失败。要从另一条分支启动运行（例如 CI 失败的分支），请按分支名检出它，例如在 `pull_request` 事件中使用 `ref: ${{ github.head_ref }}`。默认的 `pull_request` checkout 是 detached merge commit，`workflow start` 会拒绝它。TeamAI 不安装触发器，也不等待 CI 或合并。 ## 在另一台机器上继续运行 @@ -286,14 +290,18 @@ teamai workflow resume # 在另一台机器上，位于同一个仓 ## 运行记录 -结束记录包含 `prs`：通过校验的 `changes.pr_url` 按首次出现顺序排列，跨 item 和循环 pass 去重。步骤 outputs 中的 URL 不会发布。`prs` 遵循相同的 usage-report 退出设置；团队仓库公开时它也公开。没有 `prs` 的旧记录仍可读取。 - 开启[使用统计上报](reporting.md)时，每次运行都会向团队的 `teamai-reports` 分支发布一条保护隐私的记录：开始时一次，结束时一次。关闭后不发布任何记录，返回中也不带 `record`。记录存放在 `workflows/runs/.json`，以 run id 为键，重试的发布会更新同一条记录而不是产生副本。 -记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数，以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 +记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数、运行交付的 pull request（`prs`），以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 条目的 `agent` 是该 pass 实际运行时观察到的组合：步骤的 `agent:` 定义及工具如何拿到它（工具自己读取 agent 文件时为 `main`，指令拼接到提示词前时为 `prompt`）、工具及其版本、启动时请求的模型和 effort、工具报告的模型、检出最近一次 pull 的团队提交以及之后是否改过已下发的文件、项目提交、TeamAI 版本和驱动的 kind。没有观察到的值不写。`fingerprint` 对定义、工具、模型、effort 和团队提交做哈希，fingerprint 相同的两个 pass 记录下来的这些部分相同。模型或团队提交未知，或改过已下发的文件时，不写 `fingerprint`。没有任何观察的 pass，以及旧版 TeamAI 写的记录，只带定义的名称。步骤运行期间，如果该检出最近一次 pull 已完整下发到步骤启动的工具，该工具以及它在同一检出中启动的会话都会跳过会话开始时的 pull，步骤因此保持它记录的团队提交。 +`prs` 列出运行交付的每个 pull request，无论是 TeamAI 开的还是驱动上报的，每个只列一次，按运行首次交付它们的顺序排列。步骤输出中的 URL 从不列入。和记录的其他部分一样，`prs` 遵循 usage-report 的退出设置，团队仓库公开时它也公开。没有 `prs` 的旧记录仍可读取。要查找开出某个 pull request 的运行，在团队仓库 `teamai-reports` 分支的检出中运行： + +```text +jq -r 'select((.prs // []) | index("")) | .run' workflows/runs/*.json +``` + 结束运行的操作在锁内排队保存结束记录，在释放锁后发布。发布失败时——没有远端、reports 锁被占用、push 被拒——记录会进入当前 scope 数据目录的本地队列，在下一次 `teamai pull` 时重试。此时 `start`、`next` 的返回会带 `record: pending` 而不是 `record: published`。运行本身不受影响：一台机器在运行开始后崩溃，该运行会以"已开始未结束"的状态保持可见。 Run id 是 `r` 加 16 位十六进制。此方案之前创建的运行用 8 位十六进制，仍然可以解析。 diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 67e2c5cc0..8f9a12ff9 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -234,14 +234,14 @@ Generated: do not edit by hand. Regenerate with - `--json ` — Driver identity {"driver":{"kind":"ide","machine":"host","workspace":"/work"}} declares an answer a person gave in the driver UI, never one its agent chose. Records channel driver with agent false; accepts required HITL requests. - `--note ` — Keep a note with the answer, readable as steps..note - `--pass ` — Answer only this positive pass number; rejects an answered or replaced HITL request - - `teamai workflow retry ` — Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope. + - `teamai workflow retry ` — Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. A failed pull request delivery is retried in the same pass, without running the step again. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope. - `teamai workflow resume ` — Continue a run with the local driver after retry, respond, or a stopped driver (Ctrl-C or closed terminal). Uses this machine's open local lease workspace, then the latest local driver's workspace on this machine, then cwd. Another live local driver returns driver_running. A settled run returns done. A run this machine does not have is taken from teamai-reports, where workflow publish put it; a published run continues only while this install owns it, else not_owner names the owner. - `--tools ` — Override local prompt tools for this run, as comma-separated name=tool pairs - `--auto-approve` — Automatically answer optional approve HITL requests and choose defaults in this process - `--claim` — Take the published run over from the install that owns it, replacing this machine's copy and releasing its leases - `teamai workflow publish ` — Publish a run's state to teamai-reports so another member or a runner can continue it with workflow resume. The first publish makes this install the run's owner; publishing an unchanged run writes nothing. Refuses a run with records not yet on teamai-reports, or with a value that looks like a secret. Prints one JSON envelope listing the leases still in flight. - `--sessions` — Also put the agent sessions that the run's unsettled steps continue (session: continue or session.from) in the team session archive, scrubbed, so whoever continues the run resumes them; listed in the envelope as sessions. Without it, those steps start new sessions on another machine - - `teamai workflow cancel [step]` — Cancel a whole run, or a pending, waiting, leased, or running step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope. + - `teamai workflow cancel [step]` — Cancel a whole run, or a pending, waiting, leased, running, or delivering step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope. ## score diff --git a/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts index 59535baf7..a8a439a41 100644 --- a/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts +++ b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts @@ -1,60 +1,42 @@ +/** + * Real CLI: the "Start from CI" job in docs/guide/workflow.md, run the way GitHub Actions runs it. The test reads the + * job's checkout ref, `env` and `run:` commands from the guide, fills in the Actions expressions, and runs each command + * with bash in a clone of the product, where `npx teamai-cli` is the built CLI and `gh` is the fake every delivery test + * shares. The product's origin is GitHub, or a host TeamAI reaches through git alone, and the team repo is on a generic + * git host; git rewrites each to a bare repository. + */ +import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { ciRecipesOf, runCommandsOf, type CiRecipe } from '../helpers/ci-recipe.js'; import { trackDetachedProcesses } from '../helpers/detached-processes.js'; -import type { RunState } from '../../workflow/types.js'; - -const cli = fileURLToPath(new URL('../../../dist/index.js', import.meta.url)); - -describe('workflow PR delivery CLI (#1090)', () => { - it('runs the documented CI path, creates/reuses a provider PR, retains generic-git work, and refuses required unattended approvals', async () => { - const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-pr-cli-'))); - const home = path.join(root, 'home'); const product = path.join(root, 'product'); const team = path.join(root, 'team'); const bin = path.join(root, 'bin'); - for (const dir of [home, product, team, bin]) fs.mkdirSync(dir); - const detached = trackDetachedProcesses(root); - const env: NodeJS.ProcessEnv = { ...process.env, HOME: home, USERPROFILE: home, PATH: `${bin}${path.delimiter}${process.env.PATH}`, FORCE_COLOR: '0', - GIT_CONFIG_GLOBAL: path.join(home, '.gitconfig'), GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', - GIT_AUTHOR_NAME: 'Fixture', GIT_AUTHOR_EMAIL: 'fixture@example.invalid', GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.invalid', - NODE_OPTIONS: [process.env.NODE_OPTIONS, detached.nodeOptions].filter(Boolean).join(' '), PR_FIXTURE: path.join(root, 'pr.json') }; - delete env.CLAUDE_CONFIG_DIR; delete env.CODEX_HOME; delete env.GITHUB_TOKEN; delete env.GH_TOKEN; - const run = (command: string, args: string[], cwd = product) => spawnSync(command, args, { cwd, env, encoding: 'utf8', timeout: 30_000 }); - const git = (args: string[], cwd = product) => { const result = run('git', args, cwd); expect(result.status, result.stderr).toBe(0); return result.stdout.trim(); }; - const call = (args: string[]) => run(process.execPath, [cli, ...args]); - const envelope = (stdout: string) => JSON.parse(stdout.trim().split('\n').filter((line) => line.startsWith('{')).at(-1)!); - const productUrl = 'https://github.com/fixture/app.git'; const teamUrl = 'https://team.example.invalid/fixture/team.git'; - const bare = path.join(root, 'product.git'); - fs.writeFileSync(path.join(bin, 'gh'), `#!${process.execPath} -const fs = require('fs'); const args = process.argv.slice(2); const file = process.env.PR_FIXTURE; -if (args[0] === '--version') { console.log('gh version 2.80.0'); process.exit(0); } -if (args[0] === 'api') { - const saved = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null; - console.log(JSON.stringify(args[1].includes('?') ? saved ? [saved] : [] : saved)); process.exit(0); -} -if (args[0] === 'pr' && args[1] === 'create') { - const val = key => args[args.indexOf(key) + 1]; - if (val('-R') !== 'fixture/app' || val('-B') !== 'main') process.exit(7); - const pr = { state: 'open', html_url: 'https://github.com/fixture/app/pull/41', head: { ref: val('-H'), repo: { full_name: 'fixture/app' } }, base: { ref: val('-B'), repo: { full_name: 'fixture/app' } } }; - fs.writeFileSync(file, JSON.stringify(pr)); fs.appendFileSync(file + '.creates', 'create\\n'); console.log(pr.html_url); process.exit(0); -} -if (args[0] === 'pr' && args[1] === 'list') { console.log('[]'); process.exit(0); } -console.error('Unexpected provider operation: ' + args[0]); process.exit(9); -`, { mode: 0o755 }); - try { - git(['init', '-q', '-b', 'main']); fs.writeFileSync(path.join(product, 'README.md'), '# Product\n'); git(['add', '.']); git(['commit', '-qm', 'base']); - git(['init', '-q', '--bare', '-b', 'main', bare]); git(['remote', 'add', 'origin', productUrl]); - git(['config', '--global', `url.${bare}.insteadOf`, productUrl]); git(['push', '-qu', 'origin', 'main']); - git(['init', '-q', '-b', 'main'], team); fs.mkdirSync(path.join(team, 'workflows')); - fs.writeFileSync(path.join(team, 'teamai.yaml'), `team: fixture\nrepo: ${teamUrl}\nprovider: git\nusageReport: false\n`); - const yaml = `schemaVersion: v1alpha1 +import { fakeGhHost, type FakeGh } from '../helpers/fake-gh.js'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const CLI = path.join(ROOT, 'dist', 'index.js'); +const GUIDE = path.join(ROOT, 'docs', 'guide', 'workflow.md'); +const TEAM_URL = 'https://git.example.internal/acme/team.git'; +const GITHUB_URL = 'https://github.com/fixture/app.git'; +const GENERIC_URL = 'https://git.example.internal/acme/app.git'; +const ISSUE = 42; +/** What each Actions expression in the recipe stands for in the fixture. */ +const EXPRESSIONS: Record = { + 'github.event.issue.number': String(ISSUE), + 'vars.TEAM_REPO': TEAM_URL, + 'secrets.WORKFLOW_TOKEN': 'fixture-token', +}; + +const workflow = (required: boolean): string => `schemaVersion: v1alpha1 name: ship-issue -description: Deliver an issue change for review. +description: Write the issue number on a branch and deliver it for review. inputs: { issue: number } steps: - id: approve - approve: { message: Start the fixture?, required: false } + approve: { message: Ship it?, required: ${required} } - id: work needs: [approve] workspace: new @@ -63,40 +45,213 @@ steps: env: ISSUE: "{{ inputs.issue }}" `; - fs.writeFileSync(path.join(team, 'workflows', 'ship-issue.yaml'), yaml); - fs.writeFileSync(path.join(team, 'workflows', 'required.yaml'), yaml.replace('name: ship-issue', 'name: required').replace('required: false', 'required: true')); - git(['add', '.'], team); git(['commit', '-qm', 'team workflows'], team); - git(['config', '--global', `url.${team}.insteadOf`, teamUrl]); - const initialized = call(['init', teamUrl, '--scope', 'workspace', '--agent', 'claude', '--force']); expect(initialized.status, `${initialized.stdout}\n${initialized.stderr}`).toBe(0); - const pulled = call(['pull']); expect(pulled.status, pulled.stderr).toBe(0); await detached.waitForExit(); - const started = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":42}']); - expect(started.status, `${started.stdout}\n${started.stderr}`).toBe(0); - const done = envelope(started.stdout).data.done; expect(done.outcome).toBe('succeeded'); - const runFile = fs.readdirSync(root, { recursive: true, encoding: 'utf8' }).find((file) => file.endsWith(path.join('workflows', 'runs', `${done.run}.json`))); - expect(runFile).toBeDefined(); - const dataHome = path.dirname(path.dirname(path.dirname(path.join(root, runFile!)))); - const read = (id: string) => JSON.parse(fs.readFileSync(path.join(dataHome, 'workflows', 'runs', `${id}.json`), 'utf8')) as RunState; - const state = read(done.run); const work = state.steps.find((step) => step.id === 'work'); - expect(work).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/41' } }); + +const DoneSchema = z.object({ data: z.object({ done: z.object({ run: z.string(), outcome: z.string() }) }) }); +const RunSchema = z.object({ + steps: z.array(z.object({ + id: z.string(), status: z.string(), + changes: z.object({ to: z.string() }).optional(), + failure: z.object({ code: z.string(), message: z.string() }).optional(), + })), + deliveries: z.array(z.object({ url: z.string(), branch: z.string(), base: z.string(), step: z.string(), pass: z.number() })).optional(), +}); +const RecordSchema = z.object({ run: z.string(), prs: z.array(z.string()).optional() }); +const OpenSchema = z.array(z.object({ html_url: z.string(), head: z.object({ ref: z.string() }) })); + +interface Ran { + command: string; + status: number | null; + stdout: string; + stderr: string; +} + +/** Each fixture root, and a wait for the detached processes its commands left running, which still write there. */ +const fixtures: Array<{ root: string; settled: () => Promise }> = []; + +afterAll(async () => { + for (const { root, settled } of fixtures.splice(0)) { + await settled(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function git(cwd: string, args: string[], env: NodeJS.ProcessEnv = process.env): string { + const result = spawnSync('git', args, { cwd, env, encoding: 'utf8' }); + if (result.status !== 0) throw new Error(`git ${args.join(' ')} failed:\n${result.stderr}`); + return result.stdout.trim(); +} + +/** `text` with each `${{ }}` replaced as Actions does before the shell runs. */ +function expand(text: string): string { + return text.replace(/\$\{\{\s*(.+?)\s*\}\}/g, (_, expression: string) => { + const value = EXPRESSIONS[expression]; + if (value === undefined) throw new Error(`the recipe uses \${{ ${expression} }}, which the fixture does not fill in`); + return value; + }); +} + +function seed(root: string, bare: string, files: Record): void { + const dir = fs.mkdtempSync(path.join(root, 'seed-')); + git(root, ['init', '-q', '--bare', '-b', 'main', bare]); + git(dir, ['init', '-q', '-b', 'main']); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), text); + } + git(dir, ['add', '-A']); + git(dir, ['-c', 'user.name=Seed', '-c', 'user.email=seed@example.com', 'commit', '-qm', 'seed']); + git(dir, ['push', '-q', bare, 'main']); +} + +/** + * The run state `run` left in the data home of the checkout's workspace scope, `~/.teamai/projects//`, not + * the copy of its record in that data home's reports worktree. + */ +function runState(root: string, run: string): z.infer { + const state = new RegExp(`^home/\\.teamai/projects/[^/]+/workflows/runs/${run}\\.json$`); + const file = fs.readdirSync(root, { recursive: true, encoding: 'utf8' }).find((name) => state.test(name.split(path.sep).join('/'))); + if (!file) throw new Error(`no run state for ${run} under ${root}`); + return RunSchema.parse(JSON.parse(fs.readFileSync(path.join(root, file), 'utf8'))); +} + +/** + * Runs the recipe's one job against a fresh product whose origin is `origin`, with a team repo whose `ship-issue` + * workflow asks an approval that is `required` or optional. A step that fails stops the job, as in Actions. + */ +function runRecipe(origin: string, required: boolean): { recipe: CiRecipe; ran: Ran[]; root: string; gh: FakeGh; team: string; product: string } { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-ci-recipe-'))); + const detached = trackDetachedProcesses(root); + fixtures.push({ root, settled: detached.waitForExit }); + const home = path.join(root, 'home'); + const bin = path.join(root, 'bin'); + const team = path.join(root, 'team.git'); + const product = path.join(root, 'app.git'); + for (const dir of [home, bin, path.join(home, '.gh')]) fs.mkdirSync(dir); + const gh = fakeGhHost(path.join(root, 'gh')); + fs.writeFileSync(path.join(bin, 'npx'), [ + '#!/bin/sh', + '[ "$1" = teamai-cli ] || { echo "the recipe ran npx $*" >&2; exit 97; }', + 'shift', + `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(CLI)} "$@"`, '', + ].join('\n'), { mode: 0o755 }); + fs.writeFileSync(path.join(home, '.gitconfig'), [ + '[user]', '\tname = runner', '\temail = runner@example.com', + `[url "file://${team}"]`, `\tinsteadOf = ${TEAM_URL}`, + `[url "file://${product}"]`, `\tinsteadOf = ${origin}`, '', + ].join('\n')); + seed(root, team, { 'teamai.yaml': `team: acme\nrepo: ${TEAM_URL}\n`, 'workflows/ship-issue.yaml': workflow(required) }); + seed(root, product, { 'README.md': '# app\n' }); + + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: home, USERPROFILE: home, GIT_CONFIG_GLOBAL: path.join(home, '.gitconfig'), GH_CONFIG_DIR: path.join(home, '.gh'), + PATH: [bin, path.dirname(gh.bin), process.env.PATH ?? ''].join(path.delimiter), + GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', FORCE_COLOR: '0', + NODE_OPTIONS: [process.env.NODE_OPTIONS, detached.nodeOptions].filter(Boolean).join(' '), + }; + for (const name of ['XDG_CONFIG_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'GITHUB_TOKEN', 'GH_TOKEN']) delete env[name]; + + const [recipe, ...others] = ciRecipesOf(GUIDE); + if (!recipe || others.length > 0) throw new Error(`${GUIDE} shows ${others.length + (recipe ? 1 : 0)} GitHub Actions workflows, not one`); + const [job, ...jobs] = Object.values(recipe.jobs); + if (!job || jobs.length > 0) throw new Error('the recipe has more than one job'); + const checkout = job.steps.find((step) => step.uses?.startsWith('actions/checkout@')); + const ref = z.string().parse(checkout?.with?.ref); + const work = path.join(root, 'work'); + git(root, ['clone', '-q', '-b', ref, origin, work], env); + const jobEnv = Object.fromEntries(Object.entries(job.env ?? {}).map(([name, value]) => [name, expand(value)])); + + const ran: Ran[] = []; + for (const command of runCommandsOf(recipe)) { + const result = spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', expand(command)], { + cwd: work, env: { ...env, ...jobEnv }, encoding: 'utf8', timeout: 120_000, + }); + ran.push({ command, status: result.status, stdout: result.stdout, stderr: result.stderr }); + if (result.status !== 0) break; + } + return { recipe, ran, root, gh, team, product }; +} + +function said(ran: Ran[]): string { + return ran.map((step) => `$ ${step.command}\nexit ${step.status}\n${step.stdout}\n${step.stderr}`).join('\n'); +} + +function doneOf(step: Ran | undefined): { run: string; outcome: string } { + const line = step?.stdout.trim().split('\n').filter((item) => item.startsWith('{')).at(-1); + if (!line) throw new Error(`no JSON envelope from ${step?.command}`); + return DoneSchema.parse(JSON.parse(line)).data.done; +} + +function createsOf(gh: FakeGh): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr' && call.args[1] === 'create') + .map((call) => ['-R', '-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +/** The query the guide's run records section gives for the run that opened `url`, run over the records on `team`. */ +function runThatOpened(root: string, team: string, url: string): string { + const query = /^jq -r .+ workflows\/runs\/\*\.json$/m.exec(fs.readFileSync(GUIDE, 'utf8'))?.[0]; + if (!query) throw new Error(`${GUIDE} shows no jq query over workflows/runs`); + const reports = path.join(root, 'reports'); + fs.mkdirSync(path.join(reports, 'workflows', 'runs'), { recursive: true }); + for (const file of git(root, ['--git-dir', team, 'ls-tree', '--name-only', 'teamai-reports', 'workflows/runs/']).split('\n').filter(Boolean)) { + fs.writeFileSync(path.join(reports, file), git(root, ['--git-dir', team, 'show', `teamai-reports:${file}`])); + } + const result = spawnSync('bash', ['-c', query.replace('', url)], { cwd: reports, encoding: 'utf8' }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); +} + +const hasJq = spawnSync('jq', ['--version']).status === 0; + +describe('the CI recipe in the workflow guide, run through the real CLI', () => { + describe('on a GitHub origin', () => { + let job: ReturnType; + let done: { run: string; outcome: string }; + const url = 'https://github.com/fixture/app/pull/1'; + beforeAll(() => { + job = runRecipe(GITHUB_URL, false); + expect(job.ran.map((step) => step.status), said(job.ran)).toEqual(runCommandsOf(job.recipe).map(() => 0)); + done = doneOf(job.ran.at(-1)); + }, 180_000); + + it('opens one pull request from the delivered branch into the checked-out branch, and lists it in the run record', () => { + const { root, gh, team, product } = job; + expect(done.outcome).toBe('succeeded'); const branch = `teamai/${done.run}/work/p1`; - expect(git(['--git-dir', bare, 'show', `${branch}:issue.txt`])).toBe('42'); expect(fs.existsSync(path.join(product, 'issue.txt'))).toBe(false); - // Re-enter durable post-agent delivery as after a crash: the open PR is reused, with no new agent work or create. - if (work?.status !== 'succeeded') throw new Error('missing successful work'); - const persisted = state as unknown as { status: string; steps: Array> }; - persisted.status = 'running'; const target = persisted.steps.find((step) => step.id === 'work')!; target.status = 'delivering'; delete target.outcome; - delete (target.changes as Record).pr_url; delete (target.changes as Record).prRecordedAt; - fs.writeFileSync(path.join(dataHome, 'workflows', 'runs', `${done.run}.json`), JSON.stringify(persisted)); - const resumed = call(['workflow', 'resume', done.run]); expect(resumed.status, resumed.stderr).toBe(0); - expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); - git(['remote', 'set-url', 'origin', bare]); - const generic = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":43}']); - expect(generic.status).toBe(1); const genericDone = envelope(generic.stdout).data.done; - const failed = read(genericDone.run).steps.find((step) => step.id === 'work'); const failedBranch = `teamai/${genericDone.run}/work/p1`; - expect(failed).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(failedBranch) } }); - expect(git(['--git-dir', bare, 'show', `${failedBranch}:issue.txt`])).toBe('43'); - const required = call(['workflow', 'start', 'required', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":44}']); - expect(required.status).toBe(1); expect(read(envelope(required.stdout).data.done.run).steps[0]).toMatchObject({ failure: { code: 'hitl_required' } }); - expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); - } finally { await detached.waitForExit(); fs.rmSync(root, { recursive: true, force: true }); } + expect(createsOf(gh)).toEqual([['fixture/app', branch, 'main']]); + const opened = OpenSchema.parse(JSON.parse(fs.readFileSync(path.join(path.dirname(gh.bin), 'open.json'), 'utf8'))); + expect(opened.map((pull) => [pull.head.ref, pull.html_url])).toEqual([[branch, url]]); + expect(runState(root, done.run).deliveries).toEqual([{ url, branch, base: 'main', step: 'work', pass: 1 }]); + expect(git(root, ['--git-dir', product, 'show', `${branch}:issue.txt`])).toBe(String(ISSUE)); + const record = RecordSchema.parse(JSON.parse(git(root, ['--git-dir', team, 'show', `teamai-reports:workflows/runs/${done.run}.json`]))); + expect(record.prs).toEqual([url]); + }); + + it.skipIf(!hasJq)('finds the run that opened the pull request with the jq query of the guide', () => { + expect(runThatOpened(job.root, job.team, url)).toBe(done.run); + }); }); + + it('fails a run whose origin TeamAI reaches through git alone, naming the branch it pushed there', () => { + const { recipe, ran, root, gh, product } = runRecipe(GENERIC_URL, false); + + expect(ran.map((step) => step.status), said(ran)).toEqual(runCommandsOf(recipe).map((_, index, all) => (index === all.length - 1 ? 1 : 0))); + const done = doneOf(ran.at(-1)); + expect(done.outcome).toBe('failed'); + const branch = `teamai/${done.run}/work/p1`; + const work = runState(root, done.run).steps.find((step) => step.id === 'work'); + expect(work?.failure).toMatchObject({ code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) }); + expect(git(root, ['--git-dir', product, 'rev-parse', `refs/heads/${branch}`])).toBe(work?.changes?.to); + expect(createsOf(gh)).toEqual([]); + }, 180_000); + + it('fails a required approval unattended with hitl_required, and opens nothing', () => { + const { recipe, ran, root, gh } = runRecipe(GITHUB_URL, true); + + expect(ran.map((step) => step.status), said(ran)).toEqual(runCommandsOf(recipe).map((_, index, all) => (index === all.length - 1 ? 1 : 0))); + const done = doneOf(ran.at(-1)); + expect(done.outcome).toBe('failed'); + expect(runState(root, done.run).steps.find((step) => step.id === 'approve')?.failure).toMatchObject({ code: 'hitl_required' }); + expect(createsOf(gh)).toEqual([]); + }, 180_000); }); diff --git a/src/__tests__/helpers/ci-recipe.ts b/src/__tests__/helpers/ci-recipe.ts new file mode 100644 index 000000000..3415ab41e --- /dev/null +++ b/src/__tests__/helpers/ci-recipe.ts @@ -0,0 +1,26 @@ +import fs from 'node:fs'; +import { parseDocument } from 'yaml'; +import { z } from 'zod'; + +const RecipeSchema = z.object({ + jobs: z.record(z.object({ + 'runs-on': z.string().optional(), + env: z.record(z.string()).optional(), + steps: z.array(z.object({ uses: z.string().optional(), with: z.record(z.unknown()).optional(), run: z.string().optional() })), + })), +}); +export type CiRecipe = z.infer; + +/** Each GitHub Actions workflow the page at `file` shows: a yaml fence whose document has `jobs`. */ +export function ciRecipesOf(file: string): CiRecipe[] { + const text = fs.readFileSync(file, 'utf8'); + return [...text.matchAll(/^```yaml\n([\s\S]*?)^```/gm)].flatMap((match) => { + const value: unknown = parseDocument(match[1] ?? '').toJS(); + return value && typeof value === 'object' && 'jobs' in value ? [RecipeSchema.parse(value)] : []; + }); +} + +/** The `run:` commands of every job in `recipe`, in order. */ +export function runCommandsOf(recipe: CiRecipe): string[] { + return Object.values(recipe.jobs).flatMap((job) => job.steps.flatMap((step) => (step.run === undefined ? [] : [step.run]))); +} diff --git a/src/__tests__/workflow-docs-examples.test.ts b/src/__tests__/workflow-docs-examples.test.ts index 2fd399e24..062729f1f 100644 --- a/src/__tests__/workflow-docs-examples.test.ts +++ b/src/__tests__/workflow-docs-examples.test.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; import { parseDocument } from 'yaml'; import { syncScorerDefinitions } from '../scorers/cache.js'; +import { ciRecipesOf, runCommandsOf } from './helpers/ci-recipe.js'; import { formatScorerDiagnostic } from '../scorers/definition.js'; import type { LocalConfig } from '../types.js'; import { formatUnknownScorer, syncWorkflowDefinitions } from '../workflow/cache.js'; @@ -309,6 +310,19 @@ describe('workflow guide examples', () => { expect(JSON.parse(prompt.slice(prompt.indexOf('[')))).toEqual(shown); }); + it.each(PAGES)('%s: every job of a GitHub Actions workflow it shows says what it runs on', (page) => { + const jobs = ciRecipesOf(path.join(ROOT, page)).flatMap((recipe) => Object.entries(recipe.jobs)); + expect(jobs.filter(([, job]) => !job['runs-on']).map(([name]) => `${page}: job ${name} has no runs-on`)).toEqual([]); + }); + + it('shows the same CI recipe commands in every language', () => { + const recipes = PAGES.filter((page) => path.posix.basename(page) === 'workflow.md') + .map((page) => ({ page, commands: ciRecipesOf(path.join(ROOT, page)).map(runCommandsOf) })); + expect(recipes.map(({ page }) => page)).toEqual(['docs/guide/workflow.md', 'docs/guide/zh-CN/workflow.md']); + expect(recipes[0]?.commands.flat().length, 'the English guide shows a CI recipe with run commands').toBeGreaterThan(0); + for (const { page, commands } of recipes.slice(1)) expect(commands, page).toEqual(recipes[0]?.commands); + }); + it.each([...PAGES, ...SCORER_PAGES])('%s: every relative link to a .md file resolves', (page) => { const text = fs.readFileSync(path.join(ROOT, page), 'utf8'); const missing = [...text.matchAll(/\]\(([^)#\s:]+\.md)(?:#[^)]*)?\)/g)] diff --git a/src/index.ts b/src/index.ts index 723d4a0af..deb0f910e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -840,7 +840,7 @@ workflowCmd workflowCmd .command('retry ') - .description('Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope.') + .description('Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. A failed pull request delivery is retried in the same pass, without running the step again. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope.') .action(async (run: string, step: string) => { const { workflowRetry } = await import('./workflow-commands.js'); await workflowRetry(run, step); @@ -868,7 +868,7 @@ workflowCmd workflowCmd .command('cancel [step]') - .description('Cancel a whole run, or a pending, waiting, leased, or running step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope.') + .description('Cancel a whole run, or a pending, waiting, leased, running, or delivering step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope.') .action(async (run: string, step?: string) => { const { workflowCancel } = await import('./workflow-commands.js'); await workflowCancel(run, step); From 176988cede277fc278471fd5d41f34684041415c Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 16:15:03 +0200 Subject: [PATCH 27/77] test(workflow): pin that a stopped create is only looked up when the host holds none A step cancelled while TeamAI asked its host to open a pull request keeps delivery.creating, and a later next looks the request up instead of opening another. No test covered the case where the host holds no such request: the gate of that lookup could open, create a second request, and every test still passed. The new test cancels inside the create, stops before the host opened anything, and checks that the next call opens none, records no receipt, and clears the mark. Red: with the lookup-only gate always open, "only looks up the request a cancelled delivery asked for before its process stopped, and opens none when the host holds none" failed with `expected "createPullRequest" to be called 1 times, but got 2 times`. --- src/__tests__/workflow-delivery.test.ts | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 973aab6c8..c89d460b5 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -816,4 +816,31 @@ describe('retrying and cancelling a delivery', () => { expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); expect((await state(config, runId)).deliveries).toHaveLength(1); }); + + it('only looks up the request a cancelled delivery asked for before its process stopped, and opens none when the host holds none', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async () => undefined, async () => { + if (create.mock.calls.length > 1) return 'https://github.com/fixture/app/pull/9'; + await cancelRun(config, runId); + opened(); + // The process stops here, before the host opened anything. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch, creating: true } }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(create).toHaveBeenCalledTimes(1); + expect(run.deliveries).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect(run.steps[0]).not.toMatchObject({ delivery: { creating: true } }); + }); }); From 77e7f7fb224ac7f07338a24f4f42861cd046ee13 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 16:22:48 +0200 Subject: [PATCH 28/77] test(init): mock the TGit open merge request lookup TGitProvider reads tgitFindOpenMr when it is constructed, so the gf-cli mock of the init tests must export it. Red: 48 tests of init.test.ts failed with 'No "tgitFindOpenMr" export is defined on the "../providers/tgit/gf-cli.js" mock'. --- src/__tests__/init.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/__tests__/init.test.ts b/src/__tests__/init.test.ts index 8b440bb4a..1a9525de1 100644 --- a/src/__tests__/init.test.ts +++ b/src/__tests__/init.test.ts @@ -84,6 +84,7 @@ vi.mock('../providers/tgit/gf-cli.js', () => { ensureGfInstalled: () => mockEnsureGfInstalled(), ensureAuthenticated: vi.fn().mockReturnValue('testuser'), isGfInstalled: vi.fn().mockReturnValue(true), + tgitFindOpenMr: vi.fn(), RepoNotFoundError, }; }); From 1caecba19ca2875c5ea0b3550c0f166d1985596d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:02:54 +0200 Subject: [PATCH 29/77] fix(workflow): deliver a branch only while origin holds the commit that was checked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pull request holds whatever origin's branch holds when it is opened, and the protected-path check reads the commit the step reported. A driver that pushed a child of that commit, one that changes CI, got it into a pull request the check never saw, or into an accepted reported URL. Delivery now reads origin's tip of the branch with git ls-remote from the run's checkout, outside the run lock and with the delivery timeout, before it accepts a reported URL, looks up or opens anything. A tip that is not the reported commit fails the step with delivery_failed, naming the branch, saying it moved after the report or could not be read, and giving the retry command. Nothing is opened or accepted. git() in worktree.ts takes its stdin input or a timeout as one options object. Red: "opens and accepts nothing when origin's branch moved past the checked commit to one that changes protected paths" failed in both cases with `expected { data: { lease: { …(6) }, …(1) } } to match object { Object (data) }`: the step succeeded and the dependent was leased. --- src/__tests__/workflow-delivery.test.ts | 36 ++++++++++++++++++++++++- src/workflow/pr-delivery.ts | 26 ++++++++++++++++++ src/workflow/worktree.ts | 11 ++++---- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index c89d460b5..96f5b8aa5 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -99,13 +99,24 @@ function pushWork(root: string, lease: Lease, files: Record = { const clone = fs.mkdtempSync(path.join(root, 'driver-')); git(root, ['clone', '-q', path.join(root, 'remote.git'), clone]); git(clone, ['checkout', '-q', '-b', lease.workspace.branch, lease.workspace.base.commit]); + return commitAndPush(clone, lease.workspace.branch, files); +} + +/** Commits `files` on top of `branch` as origin holds it, in a clone of its own, and pushes it, as a driver that went on working does. */ +function pushOnto(root: string, branch: string, files: Record): string { + const clone = fs.mkdtempSync(path.join(root, 'driver-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + return commitAndPush(clone, branch, files); +} + +function commitAndPush(clone: string, branch: string, files: Record): string { for (const [file, content] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(clone, file)), { recursive: true }); fs.writeFileSync(path.join(clone, file), content); } git(clone, ['add', '-A']); git(clone, ['-c', 'user.name=Driver', '-c', 'user.email=driver@example.com', 'commit', '-qm', 'work']); - git(clone, ['push', '-q', 'origin', lease.workspace.branch]); + git(clone, ['push', '-q', 'origin', branch]); return git(clone, ['rev-parse', 'HEAD']); } @@ -380,6 +391,29 @@ describe('delivering a step', () => { expect(gh.calls().map((call) => call.args[0])).toEqual('pr_url' in changes ? ['api'] : []); }); + it.each([ + ['TeamAI would open the pull request', {}], + ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], + ])('opens and accepts nothing when origin\'s branch moved past the checked commit to one that changes protected paths, when %s', async (_case, changes) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { open: 'pr_url' in changes ? [pull(1, branch)] : [] }); + const checked = pushWork(root, fix, { 'src/app.ts': 'ok\n' }); + pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + expect(await succeed(config, runId, fix, { to: checked, ...changes })).not.toHaveProperty('data.dropped'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin moved after the step reported commit ${checked}`) } }, + { id: 'review', status: 'skipped' }, + ]); + expect(run.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(run.deliveries).toBeUndefined(); + expect(creates(gh)).toEqual([]); + }); + it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index a71885359..e00f5955a 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -39,6 +39,8 @@ export type DeliveryFault = | { kind: 'no_checkout' } | { kind: 'unreadable' } | { kind: 'protected_path'; files: string[] } + | { kind: 'branch_moved' } + | { kind: 'branch_unreadable' } | { kind: 'no_provider' } | { kind: 'lookup_failed'; provider: string } | { kind: 'create_failed'; provider: string } @@ -72,6 +74,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise 0) return failed({ kind: 'protected_path', files }); + // A pull request holds what origin's branch holds, so it goes out only while that is the commit just checked. A push + // after this read still reaches a request opened below; only the driver that holds the branch can make that push. + const tip = await originTip(job.checkout, job.key.branch); + if (tip.kind === 'unreadable') return failed({ kind: 'branch_unreadable' }); + if (tip.commit !== job.key.head) return failed({ kind: 'branch_moved' }); if (job.reported) return { ok: true, pr: job.reported }; const origin = await originOf(job.checkout); if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); @@ -142,6 +149,15 @@ async function findOpen( return clean.url && new URL(clean.url).hostname === hostname ? { kind: 'found', pr: clean.url } : { kind: 'unsafe' }; } +/** The commit origin's `branch` is at, read by the checkout; none when origin has no such branch. */ +async function originTip(checkout: string, branch: string): Promise<{ kind: 'read'; commit?: string } | { kind: 'unreadable' }> { + const ref = `refs/heads/${branch}`; + const listed = await git(checkout, ['ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS }); + if (listed.code !== 0) return { kind: 'unreadable' }; + const commit = listed.stdout.split('\n').map((line) => line.split('\t')).find(([, name]) => name?.trim() === ref)?.[0]; + return commit ? { kind: 'read', commit } : { kind: 'read' }; +} + /** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promise<{ files: string[] } | { fault: DeliveryFault }> { if (!OBJECT_ID.test(key.head) || !OBJECT_ID.test(base)) return { fault: { kind: 'unreadable' } }; @@ -177,6 +193,16 @@ export function deliveryFailure({ runId, key }: Pick { - if (input === undefined) { +/** Runs git in `cwd`, writing `input` to its stdin, or stopping it once `timeoutMs` passed. */ +export function git(cwd: string, args: string[], options?: { input: string } | { timeoutMs: number }): Promise { + if (options === undefined || !('input' in options)) { return new Promise((resolve) => { - execFile('git', args, { cwd, env: process.env, windowsHide: true }, (error, stdout, stderr) => { + execFile('git', args, { cwd, env: process.env, windowsHide: true, timeout: options?.timeoutMs }, (error, stdout, stderr) => { const failed = error as NodeJS.ErrnoException & { status?: number } | null; const code = typeof failed?.status === 'number' ? failed.status : failed ? 1 : 0; resolve({ code, stdout: String(stdout), stderr: String(stderr) }); @@ -85,7 +86,7 @@ export function git(cwd: string, args: string[], input?: string): Promise { stderr += chunk; }); child.on('error', () => finish(1)); child.on('close', (code) => finish(code ?? 1)); - child.stdin.end(input); + child.stdin.end(options.input); }); } @@ -490,7 +491,7 @@ async function restoreConflictStages(dir: string, conflicts: string, paths: stri // `git rm --cached` leaves stage 0 in place beside the restored stages, and then `git add` cannot clear them. const removal = paths.map((relative) => `0 ${'0'.repeat(40)}\t${relative}`); const payload = [...removal, ...lines]; - if (payload.length > 0) await git(dir, ['update-index', '--index-info'], `${payload.join('\n')}\n`); + if (payload.length > 0) await git(dir, ['update-index', '--index-info'], { input: `${payload.join('\n')}\n` }); } async function fileManifest(root: string): Promise> { From cd2d09d2fd3ddc015e472d4567eb4c6c11cbda5f Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:16:24 +0200 Subject: [PATCH 30/77] fix(workflow): keep the requests a delivery asked for on the run, not on its step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mark of a create TeamAI asked a host for lived on the step, as delivery.creating. A retry of a cancelled step, or an archive of its pass, dropped it with the pass, so a request the host opened before the process stopped was never listed. And its lookup ran as a delivery whose gate never opened: a step that a retry made deliver again meanwhile was failed by that lookup's withdrawn outcome. run.creating now lists each request, as its delivery key and target. The gate appends it under the run lock right before the create. completeDelivery removes it with the receipt of a delivered outcome, and keeps it on a failed one, since a create that failed may still have opened one. After the delivering steps, the drain looks each listed request up once with findDelivered, which reads only the origin and the provider's findOpen. Found appends the receipt, queuing the end record again on an ended run, and removes it; none removes it; failed or unsafe keeps it. That lookup touches no step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. The engine doc names its readers and writers, and the origin tip check. One sameKey compares delivery keys. Red: "records a pull request a cancelled delivery opened before its process stopped, when the step is retried before any later call" failed with `expected undefined to deeply equal [ { …(5) } ]`: the run listed no receipt. "leaves a step that was retried while a lookup-only reconcile ran delivering, and delivers it" failed with `no lease: {"data":{"done":{…,"outcome":"failed"}}}`: the lookup's withdrawn outcome failed the retried step. "looks up before it opens again …" failed with `expected undefined to deeply equal [ { step: 'fix', pass: 1, …(4) } ]`: the run listed no request. --- docs/dev/workflow-engine.md | 4 +- src/__tests__/workflow-delivery.test.ts | 109 +++++++++++++++++++-- src/workflow/engine.ts | 123 ++++++++++++++---------- src/workflow/pr-delivery.ts | 23 ++++- src/workflow/snapshot.ts | 9 +- src/workflow/step-state.ts | 8 +- src/workflow/types.ts | 6 ++ 7 files changed, 214 insertions(+), 68 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index a258fb250..2d442b149 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,8 +13,8 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreation`), runs `deliverPr` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery`, which reads the immutable `DeliveryJob` rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL or creates, looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and sets `delivery.creating`. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. -- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `completeDelivery` writes. `deliveryTarget` in `workspace.ts` reads which branches delivered, `unresolvedCreation` reads whether a step's pass has a receipt, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `delivery.creating` is set by `markCreating` and stays when the step settles or is cancelled, since `cancel` keeps `delivery`. On a step that no longer delivers, in an ended run too, `unresolvedCreation` reads it and, with no receipt for that pass, only looks the request up. `creationResolved` clears it once a lookup answered found or none. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, none removes it, failed or unsafe keeps it. `unresolvedCreate` picks each once per drain, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 96f5b8aa5..58fcf65b7 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -299,9 +299,9 @@ describe('delivering a step', () => { const run = await state(config, runId); expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); expect(gh.calls().map((call) => call.args[0])).toEqual('create' in options ? ['api', 'pr', 'api'] : ['api']); - // A step marked before it asks GitHub to open a request looks one up first after a crash; one found needs no mark. - expect(run.steps[0]).toMatchObject({ delivery: { branch, ...('create' in options ? { creating: true } : {}) } }); - if (!('create' in options)) expect(run.steps[0]).not.toHaveProperty('delivery.creating'); + // The receipt settles the request the run listed before it asked GitHub to open one. + expect(run.creating).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ delivery: { branch } }); const handed = projectRun(run, {}); expect(handed.ok && handed.value.steps[0]).toEqual(run.steps[0]); }); @@ -838,7 +838,8 @@ describe('retrying and cancelling a delivery', () => { await creating; // A stopped process leaves its delivery lock to whoever comes next. fs.rmSync(deliveryLockPath(config, runId), { force: true }); - expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch, creating: true } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ step: 'fix', pass: 1, branch, target: 'main' })]); expect((await state(config, runId)).deliveries).toBeUndefined(); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); @@ -868,13 +869,109 @@ describe('retrying and cancelling a delivery', () => { void leaseNext(config, runId, remote, writable); await creating; fs.rmSync(deliveryLockPath(config, runId), { force: true }); - expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch, creating: true } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ step: 'fix', pass: 1, branch, target: 'main' })]); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); const run = await state(config, runId); expect(create).toHaveBeenCalledTimes(1); expect(run.deliveries).toBeUndefined(); expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); - expect(run.steps[0]).not.toMatchObject({ delivery: { creating: true } }); + expect(run.creating).toBeUndefined(); + }); + + it('records a pull request a cancelled delivery opened before its process stopped, when the step is retried before any later call', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + const open: string[] = []; + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + await cancelRun(config, runId, 'fix'); + open.push(url); + opened(); + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect(await retryStep(config, runId, 'fix')).toMatchObject({ data: { retried: { step: 'fix', pass: 2 } } }); + + expect(await take(config, runId)).toMatchObject({ step: 'fix', pass: 2 }); + expect((await state(config, runId)).deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('leaves a step that was retried while a lookup-only reconcile ran delivering, and delivers it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/3'; + let retried = false; + let opened: string | undefined; + const create = providerDouble(async () => { + const step = (await state(config, runId)).steps[0]; + if (step?.status === 'failed' && !retried) { + retried = true; + expect(await retryStep(config, runId, 'fix')).toMatchObject({ data: { retried: { step: 'fix', pass: 1 } } }); + return undefined; + } + if (create.mock.calls.length === 1 && !retried) throw new Error('GitHub timed out'); + return opened; + }, async () => { + if (create.mock.calls.length === 1) throw new Error('GitHub timed out'); + opened = url; + return url; + }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(retried).toBe(true); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded' }); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(create).toHaveBeenCalledTimes(2); + }); + + it('looks up before it opens again a pull request of a step that still delivers after its process stopped inside the create', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + const url = 'https://github.com/fixture/app/pull/4'; + const asked: string[] = []; + const open: string[] = []; + let stopped = (): void => {}; + const creating = new Promise((resolve) => { stopped = resolve; }); + const create = providerDouble(async () => { + asked.push('find'); + return open[0]; + }, async () => { + asked.push('create'); + if (create.mock.calls.length > 1) { + open.push(url); + return url; + } + stopped(); + // The process stops here, before the host opened anything, and the step still delivers. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + const marked = await state(config, runId); + expect(marked.steps[0]).toMatchObject({ id: 'fix', status: 'delivering' }); + expect(marked.creating).toEqual([{ step: 'fix', pass: 1, settledAt: 2, branch, head: to, target: 'main' }]); + const handed = projectRun(marked, {}); + expect(handed.ok && handed.value.creating).toEqual(marked.creating); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(asked).toEqual(['find', 'create', 'find', 'create', 'find']); + const run = await state(config, runId); + expect(run.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect(run.creating).toBeUndefined(); }); }); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 08ca3c7fa..e0919be59 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -25,7 +25,9 @@ import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, item import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; -import { deliverPr, deliveryFailure, reportedPrMismatch, type DeliveryJob, type DeliveryKey, type DeliveryOutcome } from './pr-delivery.js'; +import { + deliverPr, deliveryFailure, findDelivered, reportedPrMismatch, type DeliveryJob, type DeliveryKey, type DeliveryOutcome, type UnresolvedCreate, +} from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { withFileLock } from '../update.js'; @@ -884,12 +886,14 @@ function deliveryKey(step: DeliveringStep): DeliveryKey { return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; } +/** Whether `a` and `b` name one delivery: the same pass of the same step, settled at the same revision, of one commit. */ +function sameKey(a: DeliveryKey, b: DeliveryKey): boolean { + return a.step === b.step && a.pass === b.pass && a.settledAt === b.settledAt && a.branch === b.branch && a.head === b.head; +} + /** The step `key` names, while it still delivers that pass of that commit. */ function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep | undefined { - const step = run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && item.id === key.step); - if (!step) return undefined; - const current = deliveryKey(step); - return current.pass === key.pass && current.settledAt === key.settledAt && current.branch === key.branch && current.head === key.head ? step : undefined; + return run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && sameKey(deliveryKey(item), key)); } /** @@ -917,42 +921,41 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { return undefined; } -/** - * A request a step asked its host to open before it settled otherwise, cancelled or failed, with no receipt for its pass: - * the host may hold one the run does not list. Its job is only looked up, never opened. `tried` holds the steps this - * drain already looked up, so one the host cannot answer for now waits for a later call. - */ -function unresolvedCreation(run: RunState, tried: ReadonlySet): DeliveryJob | undefined { - if (!run.checkout || !run.base) return undefined; - for (const step of run.steps) { - if (step.status === 'delivering' || !('delivery' in step) || !step.delivery?.creating || tried.has(step.id)) continue; - const key = { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; - if (run.deliveries?.some((receipt) => receipt.step === key.step && receipt.pass === key.pass && receipt.branch === key.branch)) continue; - const target = deliveryTarget(run, key.branch); - if (target !== undefined) return { key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target }; - } - return undefined; +/** The first request in `run.creating` that this drain has not looked up yet. */ +function unresolvedCreate(run: RunState, tried: readonly UnresolvedCreate[]): { create: UnresolvedCreate; checkout: string } | undefined { + const { checkout } = run; + const create = run.creating?.find((listed) => !tried.some((done) => sameCreate(done, listed))); + return create && checkout ? { create, checkout } : undefined; } -/** Drops the mark of the request `key` asked for, once a lookup said whether its host holds it. */ -function creationResolved(run: RunState, key: DeliveryKey): void { - const step = run.steps.find((item) => item.id === key.step && (item.pass ?? 1) === key.pass); - if (step && step.status !== 'delivering' && 'delivery' in step && step.delivery?.branch === key.branch && step.delivery.head === key.head) { - delete step.delivery.creating; - } +function sameCreate(a: UnresolvedCreate, b: UnresolvedCreate): boolean { + return sameKey(a, b) && a.target === b.target; +} + +/** Drops `create` from the requests the run has no receipt for. */ +function forgetCreate(run: RunState, create: UnresolvedCreate): void { + const left = run.creating?.filter((listed) => !sameCreate(listed, create)); + if (left?.length) run.creating = left; + else delete run.creating; +} + +/** Lists `url` as what `key` delivered into `target`, once; true when the run had no receipt for it yet. */ +function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrUrl): boolean { + if (run.deliveries?.some((receipt) => receipt.url === url)) return false; + (run.deliveries ??= []).push({ url, branch: key.branch, base: target, step: key.step, pass: key.pass }); + return true; } /** * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step - * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The step settles only - * while it still delivers what `job` delivered, at the revision its agent's report settled it. `requeueEnd` is set when - * the receipt reached a run that already ended, whose end record must go out again to list it. + * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the + * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The + * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. + * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. */ function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome): { requeueEnd: boolean } { - const receipted = outcome.ok && !run.deliveries?.some((receipt) => receipt.url === outcome.pr); - if (outcome.ok && receipted) { - (run.deliveries ??= []).push({ url: outcome.pr, branch: job.key.branch, base: job.target, step: job.key.step, pass: job.key.pass }); - } + const receipted = outcome.ok && recordReceipt(run, job.key, job.target, outcome.pr); + if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); if (run.status === 'running') reconcileRun(run); const step = deliveringStep(run, job.key); if (step && outcome.ok) delivered(step); @@ -960,10 +963,21 @@ function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutc return { requeueEnd: receipted && run.status !== 'running' }; } +/** + * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, + * and one it does not hold is dropped. One it could not answer for stays for a later drain. + */ +function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited>): { requeueEnd: boolean } { + if (found.kind === 'failed') return { requeueEnd: false }; + const receipted = found.kind === 'found' && recordReceipt(run, create, create.target, found.pr); + forgetCreate(run, create); + return { requeueEnd: receipted && run.status !== 'running' }; +} + /** * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still - * delivers `job`. If it does, it is marked as creating in the same write, so a request opened by a process that then - * stops is looked up rather than opened again. + * delivers `job`. If it does, the run lists its request in the same write, so one opened by a process that then stops + * is looked up, whatever becomes of the step, rather than lost. */ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob): Promise { const marked = await updateRun(config, runId, async (run) => { @@ -971,7 +985,8 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob const settledRun = reconcileRun(run); const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); if (!step) return { value: false }; - step.delivery.creating = true; + const create = { ...job.key, target: job.target }; + if (!run.creating?.some((listed) => sameCreate(listed, create))) (run.creating ??= []).push(create); return { value: true, write: { ...run, updatedAt: new Date().toISOString() } }; }); return marked.status === 'done' && marked.value; @@ -979,40 +994,46 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob /** * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it, then looks up - * the requests that steps which stopped delivering may have left open, on an ended run too. The run lock is held to pick - * a delivery and to apply its outcome, never across git or a provider. While another process of this machine delivers, - * it holds the delivery lock, and this call leaves the work to it. + * each request in `run.creating` once, on an ended run too. The run lock is held to pick a delivery and to apply its + * outcome, never across git or a provider. While another process of this machine delivers, it holds the delivery lock, + * and this call leaves the work to it. */ async function drainDeliveries(config: LocalConfig, runId: string): Promise { if (!isRunId(runId)) return; const loaded = await readRun(config, runId); if (typeof loaded === 'string') return; const delivering = loaded.status === 'running' && loaded.steps.some((step) => step.status === 'delivering'); - if (!delivering && !unresolvedCreation(loaded, new Set())) return; + if (!delivering && !unresolvedCreate(loaded, [])) return; const drained = await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { let requeued = false; - const tried = new Set(); + const tried: UnresolvedCreate[] = []; for (;;) { const picked = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); - const next = nextDelivery(run); - const unresolved = next ? undefined : unresolvedCreation(run, tried); - const value = next ? { job: next, lookupOnly: false } : unresolved && { job: unresolved, lookupOnly: true }; + const job = nextDelivery(run); + const value = job ? { job } : unresolvedCreate(run, tried); return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); if (picked.status !== 'done' || !picked.value) return requeued; - const { job, lookupOnly } = picked.value; - if (lookupOnly) tried.add(job.key.step); - // A gate that never opens leaves a lookup: the request is found, or `withdrawn` says the host holds none. - const outcome = await deliverPr(job, lookupOnly ? async () => false : () => markCreating(config, runId, job)); + let apply: (run: RunState) => { requeueEnd: boolean }; + if ('job' in picked.value) { + const { job } = picked.value; + const outcome = await deliverPr(job, () => markCreating(config, runId, job)); + apply = (run) => completeDelivery(run, job, outcome); + } else { + const { create, checkout } = picked.value; + tried.push(create); + const found = await findDelivered(checkout, create); + apply = (run) => resolveCreate(run, create, found); + } const applied = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; - const { requeueEnd } = completeDelivery(run, job, outcome); - if (lookupOnly && (outcome.ok || outcome.fault.kind === 'withdrawn')) creationResolved(run, job.key); + const before = JSON.stringify(run); + const { requeueEnd } = apply(run); // Queued under the lock with the receipt, as finishRun queues an end; it publishes once the lock is released. if (requeueEnd) run.endRecordQueued = await queueRunRecord(config, run, run.driver, 'end'); - return { value: requeueEnd, write: { ...run, updatedAt: new Date().toISOString() } }; + return { value: requeueEnd, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); if (applied.status !== 'done' || applied.value === undefined) return requeued; requeued ||= applied.value; diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index e00f5955a..9e5c876b3 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -19,6 +19,12 @@ export interface DeliveryKey { head: string; } +/** + * A pull request TeamAI asked a host to open, into `target`, and holds no receipt for yet. The run lists it until a + * receipt or a lookup settles it, whatever becomes of its step, since the host may hold it. + */ +export type UnresolvedCreate = DeliveryKey & { target: string }; + /** Everything a delivery needs, read from the run under its lock. */ export interface DeliveryJob { key: DeliveryKey; @@ -50,8 +56,8 @@ export type DeliveryFault = export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; /** - * Asked under the run lock right before a pull request is opened: true while the step still delivers the job, which it - * then marks as creating. False opens nothing. + * Asked under the run lock right before a pull request is opened: true while the step still delivers the job, whose + * request the run then lists as unresolved. False opens nothing. */ export type CreateGate = () => Promise; @@ -125,6 +131,19 @@ export async function reportedPrMismatch( return undefined; } +/** + * Whether the host holds the open request `create` asked for, by lookup alone: nothing of the branch is read, and + * nothing is opened. `failed` is a host that cannot answer, an origin no provider reads, or a URL that cannot be kept. + */ +export async function findDelivered( + checkout: string, { branch, target }: Pick, +): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' }> { + const origin = await originOf(checkout); + if (origin.kind === 'transport') return { kind: 'failed' }; + const found = await findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); + return found.kind === 'unsafe' ? { kind: 'failed' } : found; +} + /** The provider of the checkout's configured origin; git alone when the checkout has none. */ async function originOf(checkout: string): Promise { const remote = await git(checkout, ['config', '--get', 'remote.origin.url']); diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index f3a961d37..203d4a783 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -10,6 +10,7 @@ import { ArchivedPassSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; import type { RunState, StepLease, WorkflowDefinition, WorkflowFailure } from './types.js'; +import type { UnresolvedCreate } from './pr-delivery.js'; /** * The run state one install hands to another, at `workflows/runs//state.json` on `teamai-reports`. It carries what @@ -37,6 +38,10 @@ function withCommitIds(schema: T): z.ZodEffects { }); } +const UnresolvedCreateSchema: z.ZodType = z.object({ + step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: z.string().min(1), target: z.string().min(1), +}); + const BodySchema = z.object({ schemaVersion: z.literal('v1alpha1'), name: z.string(), @@ -56,6 +61,8 @@ const BodySchema = z.object({ /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ copies: z.record(z.object({ commit: CommitIdSchema.optional(), from: z.string().optional() })), deliveries: z.array(DeliveryReceiptSchema).optional(), + /** The requests a claimer looks up before anything opens another: the machine that handed the run over may have opened them. */ + creating: z.array(UnresolvedCreateSchema).optional(), sessionContinuations: z.array(SessionContinuationSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ scorers: z.record(z.object({ sourceHash: z.string() })).optional(), @@ -197,7 +204,7 @@ const RUN_RULES = { end: 'publish', failure: 'rewrite', inputs: 'publish', sourceHash: 'publish', teamRevision: 'publish', definition: 'rewrite', workflows: 'rewrite', scorers: 'rewrite', steps: 'rewrite', settledAttempts: 'rewrite', settledCount: 'rewrite', endRecordQueued: 'drop', cancelledAttempts: 'rewrite', archive: 'rewrite', round: 'publish', base: 'publish', - checkout: 'drop', copies: 'rewrite', deliveries: 'publish', sessionContinuations: 'publish', publication: 'drop', createdAt: 'publish', updatedAt: 'publish', + checkout: 'drop', copies: 'rewrite', deliveries: 'publish', creating: 'publish', sessionContinuations: 'publish', publication: 'drop', createdAt: 'publish', updatedAt: 'publish', } satisfies FieldRules; const STEP_RULES = { diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 6036f0b42..268c5359e 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -131,12 +131,8 @@ type StepBase = z.infer; /** * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. - * `creating` is set under the run lock right before TeamAI asks the host to open a request, and stays: a request may - * exist with no receipt after a crash or a cancel, so such a step is looked up before anything opens another. */ -const DeliverySchema = z.object({ - branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional(), creating: z.literal(true).optional(), -}); +const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); export type Delivery = z.infer; /** @@ -328,7 +324,7 @@ export function fail(state: StepState, settlement: Settlement): Fail /** * An open step is cancelled before it settles. A leased or delivering one keeps the attempt it no longer settles, and a - * delivering one what it was delivering, since a request it asked for may exist with no receipt yet. + * delivering one the branch that holds its work. */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { const attempt = state.status === 'leased' || state.status === 'delivering' ? state.attempt : undefined; diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 27ab1571a..047d443db 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,6 +1,7 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; +import type { UnresolvedCreate } from './pr-delivery.js'; import type { DeliveryReceipt } from './pr-url.js'; import type { ArchivedPass, SessionContinuation, StepState } from './step-state.js'; import type { LeaseSession } from './workspace.js'; @@ -266,6 +267,11 @@ export interface RunState { * request was opened, never that it is still open, so whether a delivery reuses one is its host's lookup to say. */ deliveries?: DeliveryReceipt[]; + /** + * The pull requests TeamAI asked a host to open and has no receipt for, listed under the run lock right before each + * create. A receipt or a lookup removes one; a retry, a cancel or an archive of its step does not. + */ + creating?: UnresolvedCreate[]; /** * Sessions a step resumed and continued under another id, in report order: a copy on another machine or in another * tool. `to` holds the turns of `from` and the ones that step added, so a step that would resume `from` resumes `to`. From 30042532a7d39dbead73719221d68198b509afee Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:17:46 +0200 Subject: [PATCH 31/77] fix(workflow): record no URL after a create that the origin host check refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a create, every lookup answer other than found fell through to the sanitized URL the create answered with. A lookup that named a request on another host was overridden by it, and a create URL on another host was recorded whenever the lookup could not answer. Both bypassed the host rule every other recorded URL passes. A lookup after the create that answers with a URL that cannot be kept is now an unsafe_url fault. When it cannot answer, the create's URL is kept only on the origin's pull request host; on any other host it is an unsafe_url fault too. The request stays listed in run.creating, so a later drain looks it up. Red: "records the URL a create answered only on the origin host, when the lookup after it lists a request on another host" and "… cannot answer, and the create named a request on another host" failed with `expected { data: { lease: { …(6) }, …(1) } } to match object { Object (data) }`: the URL was recorded and the dependent was leased. --- src/__tests__/workflow-delivery.test.ts | 23 +++++++++++++++++++++++ src/workflow/pr-delivery.ts | 5 ++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 58fcf65b7..d95dc3070 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -629,6 +629,29 @@ describe('where a pull request goes', () => { expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); }); + it.each([ + ['lists a request on another host', async (): Promise => 'https://other.example/review/1', 'https://github.com/fixture/app/pull/1', false], + ['cannot answer, and the create named a request on another host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://other.example/review/1', false], + ['cannot answer, and the create named a request on the origin host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://github.com/fixture/app/pull/1', true], + ])('records the URL a create answered only on the origin host, when the lookup after it %s', async (_case, lookupAfter, answered, kept) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const create = providerDouble(async () => (create.mock.calls.length === 0 ? undefined : lookupAfter()), async () => answered); + + if (kept) { + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual([answered]); + return; + } + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).toHaveBeenCalledTimes(1); + }); + it('starts the next for_each item while an earlier one delivers, since max_parallel bounds agents', async () => { const { config, root, runId } = await fixture(` - id: build\n for_each: { items: inputs.packages, max_parallel: 1 }\n workspace: new\n command: node build.js\n deliver: pr\n`, { packages: ['one', 'two'].map((name) => ({ name, id: name, after: [] })) }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 9e5c876b3..9fe7312d5 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -111,8 +111,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 17:24:21 +0200 Subject: [PATCH 32/77] fix(workflow): keep what the agent settled when its delivery is cancelled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cancelling a delivering step kept only its attempt and delivery. The changes, outputs, session, tool, tokens, observation and reporting install of an agent that had already succeeded were dropped, so a scorer that had not picked the pass yet never saw it, and the end record lost its token usage. A step cancelled while it delivers now keeps every settled field, as the delivering variant holds them; the cancelled variant takes the same settled fields. A pass keeps its delivery in history and in the snapshot, so agentSucceeded judges a cancelled pass that was delivering as the agent's success, before and after a retry replaces it. Red: "judges a step cancelled while it delivers as its agent left it, and the run record keeps the tokens of that pass" failed with `expected { id: 'review', …(5) } to match object { status: 'cancelled', …(2) }`: the cancelled step had lost its tokens and outputs. --- src/__tests__/workflow-scoring.test.ts | 20 ++++++++++++ src/workflow/snapshot.ts | 1 + src/workflow/step-state.ts | 43 ++++++++++++++------------ 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/src/__tests__/workflow-scoring.test.ts b/src/__tests__/workflow-scoring.test.ts index f010a5163..f412f9d60 100644 --- a/src/__tests__/workflow-scoring.test.ts +++ b/src/__tests__/workflow-scoring.test.ts @@ -16,6 +16,7 @@ import { workflowRunDetail } from '../dashboard/workflow-runs.js'; import { buildEndRecord } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; import { nextJudgeLease, parseJudgeRequest, parseJudgeSubmission, runJudgeLease, scoreOnDemand, scoredPasses, submitJudgeLease, type JudgeRequest } from '../workflow/scoring.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; import { installOwner, readRun, updateRun } from '../workflow/store.js'; import type { StepState } from '../workflow/step-state.js'; import type { DriverIdentity, RunState, WorkflowEnvelope } from '../workflow/types.js'; @@ -125,6 +126,25 @@ describe('judge work', () => { expect(await LabelStore.local(config)).toEqual(labels); }); + it('judges a step cancelled while it delivers as its agent left it, and the run record keeps the tokens of that pass', async () => { + const config = fixture(scorer(100)); + const tokens = { input: 1200, output: 80 }; + const runId = await settledRun(config, { tokens }); + const settled = (await runOf(config, runId)).steps[0]; + if (settled?.status !== 'succeeded') throw new Error('review did not succeed'); + await setStep(config, runId, (run) => ({ ...settled, status: 'delivering', delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } })); + expect(await cancelRun(config, runId, 'review')).toMatchObject({ data: { cancelled: { steps: ['review'] } } }); + + const cancelled = await runOf(config, runId); + expect(cancelled.steps[0]).toMatchObject({ status: 'cancelled', tokens, outputs: { reviewed: true } }); + expect(buildEndRecord(cancelled, workDriver, 'tester').steps).toEqual([expect.objectContaining({ step: 'review', pass: 1, outcome: 'cancelled', tokens })]); + expect(data(await nextJudgeLease(config, request()))).toHaveProperty('lease'); + + expect(await retryStep(config, runId, 'review')).toMatchObject({ data: { retried: { step: 'review', pass: 2 } } }); + const install = (await installOwner(config)).id; + expect(scoredPasses(await runOf(config, runId), install)).toEqual([{ subject: { runId, step: 'review', pass: 1 }, scorers: ['quality'] }]); + }); + it('finalizes the selected Codex tool in the lease', async () => { const config = fixture(scorer(100)); await settledRun(config); diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 203d4a783..0c4b23332 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -219,6 +219,7 @@ const PASS_RULES = { pass: 'publish', attempt: 'publish', outcome: 'publish', outputs: 'publish', end: 'publish', results: 'publish', failure: 'rewrite', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', agent: 'publish', used: 'publish', installId: 'publish', answer: 'rewrite', + delivery: 'publish', } satisfies FieldRules; const ARCHIVED_RULES = { ...PASS_RULES, step: 'publish', within: 'publish' } satisfies FieldRules; diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 268c5359e..08d241cdf 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -87,6 +87,13 @@ const AnswerSchema = z.object({ /** The answer and provenance retained for one pass of a step with HITL. */ export type HitlAnswer = z.infer; +/** + * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on + * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. + */ +const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); +export type Delivery = z.infer; + /** One earlier pass of a step instance that a loop ran again. */ export const StepPassSchema = z.object({ pass: z.number().int(), @@ -97,6 +104,8 @@ export const StepPassSchema = z.object({ results: ResultsSchema.optional(), failure: FailureSchema.optional(), ...report, + /** What the pass delivered once its agent succeeded, so a pass whose delivery failed or was cancelled is still the agent's success. */ + delivery: DeliverySchema.optional(), /** The answer retained with this archived pass. */ answer: AnswerSchema.optional(), }); @@ -128,13 +137,6 @@ const StepBaseSchema = z.object({ }); type StepBase = z.infer; -/** - * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on - * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. - */ -const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); -export type Delivery = z.infer; - /** * What a step that ran keeps. A file written before `settledAt` existed reads it as 0. `end` is the end the workflow * this step runs reached. @@ -166,10 +168,8 @@ const SucceededSchema = StepBaseSchema.extend({ }); const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional() }); const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); -const CancelledSchema = StepBaseSchema.extend({ - status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema, - delivery: DeliverySchema.optional(), -}); +/** A step cancelled while it delivered keeps what its agent settled, as `delivering` holds it. */ +const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }); const LeasedSchema = StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }); /** @@ -323,15 +323,16 @@ export function fail(state: StepState, settlement: Settlement): Fail } /** - * An open step is cancelled before it settles. A leased or delivering one keeps the attempt it no longer settles, and a - * delivering one the branch that holds its work. + * An open step is cancelled before it settles. A leased one keeps the attempt it no longer settles. A delivering one + * keeps what its agent settled, and what it delivered, since its agent succeeded whatever becomes of the delivery. */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { - const attempt = state.status === 'leased' || state.status === 'delivering' ? state.attempt : undefined; - const delivery = state.status === 'delivering' ? state.delivery : undefined; - return become(state, { - status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), ...(delivery ? { delivery } : {}), failure, - }); + if (state.status === 'delivering') { + const { id: _id, status: _status, ...settlement } = state; + return become(state, { ...settlement, status: 'cancelled', settledAt, failure }); + } + const attempt = state.status === 'leased' ? state.attempt : undefined; + return become(state, { status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), failure }); } /** The current pass of a settled step, as `passes` keeps it once a loop runs the step again; undefined while it has not settled. */ @@ -349,10 +350,12 @@ export function agentPassOf(state: StepState): StepPass | undefined { return state.status === 'delivering' ? StepPassSchema.parse({ ...state, pass: state.pass ?? 1, outcome: 'success' }) : passOf(state); } -/** Whether the agent of `pass` succeeded: it did, or only the delivery of what it wrote failed. */ +/** Whether the agent of `pass` succeeded: it did, or only the delivery of what it wrote failed or was cancelled. */ export function agentSucceeded(pass: StepPass): boolean { const code: string | undefined = pass.failure?.code; - return pass.outcome === 'success' || (pass.outcome === 'failed' && DELIVERY_FAILURE_CODES.some((delivery) => delivery === code)); + return pass.outcome === 'success' + || (pass.outcome === 'failed' && DELIVERY_FAILURE_CODES.some((delivery) => delivery === code)) + || (pass.outcome === 'cancelled' && pass.delivery !== undefined); } /** From 52500e07eaec974ecfa8d71cbf347838445b09c3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:27:01 +0200 Subject: [PATCH 33/77] fix(workflow): leave a step delivering when the run lock is busy at the create gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate before a create read a run lock it could not take within its attempts the same as a step that no longer delivers. The delivery came back withdrawn, and completion failed a step that still delivered with "the step no longer delivered", the one delivery failure that gives no retry command. The gate now answers open, refused or busy. A busy gate opens nothing, deliverPr hands it back as busy, and the drain stops for that call, as it does when the pick is busy: the step stays delivering, next answers wait.delivering, and a later call delivers it. withdrawn is left for a real refusal. Red: "leaves a step delivering, opening nothing, while the run lock is held when its pull request is about to be opened" failed with `expected { data: { done: { …(2) }, …(1) } } to deeply equal { data: { wait: { …(2) }, …(1) } }`: the busy gate failed the step and the run ended. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 38 +++++++++++++++++++++++-- src/workflow/engine.ts | 17 ++++++----- src/workflow/pr-delivery.ts | 16 +++++++---- 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 2d442b149..c8452ecf2 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, none removes it, failed or unsafe keeps it. `unresolvedCreate` picks each once per drain, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index d95dc3070..a18c05cb5 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -9,7 +9,8 @@ import { parseWorkflow } from '../workflow/parse.js'; import { leaseNext, parseReport, reportAttempt, startRun, withIgnored } from '../workflow/engine.js'; import { sanitizePrUrl } from '../workflow/pr-url.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; -import { deliveryLockPath, readRun } from '../workflow/store.js'; +import { deliveryLockPath, readRun, runLockPath } from '../workflow/store.js'; +import * as store from '../workflow/store.js'; import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; import { acquireLock, releaseLock } from '../update.js'; import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; @@ -351,7 +352,7 @@ describe('delivering a step', () => { const to = pushWork(root, fix); const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main' }; - expect(await deliverPr(job, async () => false)).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); + expect(await deliverPr(job, async () => 'refused')).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); }); @@ -806,6 +807,39 @@ describe('retrying and cancelling a delivery', () => { expect(gh.calls()).toEqual([]); }); + it('leaves a step delivering, opening nothing, while the run lock is held when its pull request is about to be opened', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const runLock = runLockPath(config, runId); + let holding = false; + let released = false; + const update = store.updateRun; + vi.spyOn(store, 'updateRun').mockImplementation(async (...args) => { + const result = await update(...args); + // The process that holds the run lock lets it go once an update gave up waiting for it. + if (result.status === 'busy' && holding) { + await releaseLock(runLock); + holding = false; + released = true; + } + return result; + }); + const create = providerDouble(async () => { + if (!released && !holding) holding = await acquireLock(runLock); + return undefined; + }, async () => 'https://github.com/fixture/app/pull/1'); + + expect(await leaseNext(config, runId, remote, writable)).toEqual({ data: { wait: { reason: 'fix is delivering', delivering: ['fix'] }, cancelled: [] } }); + expect(released).toBe(true); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'delivering' }); + expect(create).not.toHaveBeenCalled(); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(create).toHaveBeenCalledTimes(1); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + }); + it('opens nothing when the run is cancelled while its delivery looks for an open pull request', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index e0919be59..b873535bf 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -979,17 +979,17 @@ function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited { - const marked = await updateRun(config, runId, async (run) => { - if (run === 'missing' || run.status !== 'running') return { value: false }; +async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob): Promise<'open' | 'refused' | 'busy'> { + const marked = await updateRun(config, runId, async (run): Promise<{ value: 'open' | 'refused'; write?: RunState }> => { + if (run === 'missing' || run.status !== 'running') return { value: 'refused' }; const settledRun = reconcileRun(run); const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); - if (!step) return { value: false }; + if (!step) return { value: 'refused' }; const create = { ...job.key, target: job.target }; if (!run.creating?.some((listed) => sameCreate(listed, create))) (run.creating ??= []).push(create); - return { value: true, write: { ...run, updatedAt: new Date().toISOString() } }; + return { value: 'open', write: { ...run, updatedAt: new Date().toISOString() } }; }); - return marked.status === 'done' && marked.value; + return marked.status === 'done' ? marked.value : 'busy'; } /** @@ -1020,6 +1020,8 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise markCreating(config, runId, job)); + // As when the pick is busy: the step still delivers, and a later call delivers it. + if (outcome === 'busy') return requeued; apply = (run) => completeDelivery(run, job, outcome); } else { const { create, checkout } = picked.value; @@ -1073,7 +1075,8 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv return answer(doneData(run)); }; const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => { - // Steps still delivering here are being delivered by another process of this machine; a driver polls until they settle. + // Steps still delivering here are being delivered by another process of this machine, or waited for a busy run + // lock; a driver polls until they settle. const delivering = run.steps.filter((step) => step.status === 'delivering').map((step) => step.id); const reason = open ? `${open.id} is leased` : delivering.length > 0 ? `${delivering[0]} is delivering` : 'canTakeLease is false'; return answer({ wait: { reason, ...(delivering.length > 0 ? { delivering } : {}) } }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 9fe7312d5..ab9aaab00 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -56,10 +56,11 @@ export type DeliveryFault = export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; /** - * Asked under the run lock right before a pull request is opened: true while the step still delivers the job, whose - * request the run then lists as unresolved. False opens nothing. + * Asked under the run lock right before a pull request is opened: `open` while the step still delivers the job, whose + * request the run then lists as unresolved; `refused` when it no longer does; `busy` when the run lock could not be + * taken, so nobody knows yet. Only `open` opens anything. */ -export type CreateGate = () => Promise; +export type CreateGate = () => Promise<'open' | 'refused' | 'busy'>; /** How long a host may take to answer one lookup or one create. */ export const DELIVERY_TIMEOUT_MS = 30_000; @@ -72,10 +73,11 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) /** * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch - * is looked up, and only when there is none, and `gate` agrees, is one opened; then it is looked up again. + * is looked up, and only when there is none, and `gate` opens, is one opened; then it is looked up again. `busy` is a + * gate that could not answer: nothing was opened, and the job is for a later call. * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ -export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { +export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { const changed = await changedFiles(job); if ('fault' in changed) return failed(changed.fault); const files = protectedPaths(changed.files); @@ -94,7 +96,9 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise | undefined; try { created = sanitizePrUrl(await provider.createPullRequest({ From 7374ee445cf8bf39313dab27d2d563bae442a42b Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:30:21 +0200 Subject: [PATCH 34/77] fix(workflow): name a reported pull request URL that no delivery uses as dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sanitized and vetted changes.pr_url was discarded with no dropped entry when the step failed, including a success the engine failed as invalid_output, when it was released, and when its branch held nothing beyond the run base. A driver that opened its own pull request was not told TeamAI ignored it. The report envelope now names changes.pr_url in dropped in those cases, with "the step failed", "the step was released", or "the branch has no changes beyond the run base". The reason never holds the value. Both workflow guides say so. Red: "names a kept URL as dropped, without its value, when the step fails", "… succeeds with stdout that does not match its output", "… is released" and "… succeeds with its branch at the run base" failed with `expected { data: { accepted: true, …(2) } } to match object { data: { accepted: true, …(1) } }`: the envelope had no dropped entry. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 21 +++++++++++++++++++++ src/workflow/engine.ts | 9 +++++++-- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 0f968c082..e7ea5c471 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,7 +244,7 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. -A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep, or one from a step without `deliver: pr`, is dropped. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 352556d6c..3e1c8a1bd 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,7 +244,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 -自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL，以及来自未设置 `deliver: pr` 的步骤的 URL，会被丢弃。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index a18c05cb5..b3d1b45d2 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -717,6 +717,27 @@ describe('reported URLs from a driver', () => { expect(gh.calls()).toEqual([]); }); + it.each([ + ['fails', FIX_THEN_REVIEW, { outcome: 'failed' }, true, 'the step failed'], + ['succeeds with stdout that does not match its output', `${FIX_THEN_REVIEW.split(' - id: review')[0]} output:\n ok: boolean\n`, { outcome: 'success', outputs: { ok: 'yes' } }, true, 'the step failed'], + ['is released', FIX_THEN_REVIEW, { outcome: 'released' }, true, 'the step was released'], + ['succeeds with its branch at the run base', FIX_THEN_REVIEW, { outcome: 'success' }, false, 'the branch has no changes beyond the run base'], + ])('names a kept URL as dropped, without its value, when the step %s', async (_case, steps, body, pushes, reason) => { + const { config, root, runId } = await fixture(steps); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const base = fix.workspace?.base.commit; + const to = pushes ? pushWork(root, fix) : base; + const url = 'https://review.acme.internal/changes/4821'; + const accepted = await report(config, runId, { attempt: fix.attempt, revision: fix.revision, ...body, + changes: { branch: fix.workspace?.branch, base, from: base, to, commits: pushes ? [to] : [], pr_url: url } }); + + expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason }] } }); + expect(JSON.stringify(accepted)).not.toContain('changes/4821'); + expect((await state(config, runId)).deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + it.each([ ['is not the open pull request of its branch', [7], (branch: string) => `is not the open pull request of branch ${branch} into main`, 7], ['names a branch with no open pull request', [], (branch: string) => `branch ${branch} has no open pull request into main`, 1], diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index b873535bf..558e792e5 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1570,8 +1570,13 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. Delivery reads the // pushed branch, so it does not keep the worktree. const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); - // A URL that is not kept is named without its value, which may hold a token, and the report still counts. - const dropReason = reported?.reason ?? unmatched ?? (reported && defined?.deliver !== 'pr' ? 'the step has no deliver: pr' : undefined); + // A URL that is not kept, or that no delivery uses, is named without its value, which may hold a token, and the + // report still counts. + const dropReason = reported?.reason ?? unmatched ?? (reported?.url === undefined || delivery ? undefined + : defined?.deliver !== 'pr' ? 'the step has no deliver: pr' + : outcome === 'released' ? 'the step was released' + : outcome !== 'success' ? 'the step failed' + : 'the branch has no changes beyond the run base'); return { save: true, envelope: ok({ From 0c9848224246eb5f670b0d94ec91b8fd2bbe1ae4 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:33:06 +0200 Subject: [PATCH 35/77] fix(workflow): refuse deliver: pr on a workflow step whose workflow never writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deliveryProblem exempted a for_each workflow step from the workspace: new rule, and checked the mode only of steps that run no workflow. A for_each step that delivers, whose item workflow only reads, passed workflow start, and every item then succeeded with no pull request and no notice. workflow start now fails bad_delivery for a delivering workflow step, for_each or not, unless some step its workflow runs, at any depth, resolves to a writable workspace. The message names the step and the two ways to make it write. Both workflow guides say so. Red: "refuses a for_each workflow step that delivers, by whether a step its items run writes a branch" failed with `expected { data: { …(5) } } to match object { error: { …(2) } }`: the run started. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-engine.test.ts | 24 ++++++++++++++++++++++++ src/workflow/workspace.ts | 10 +++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index e7ea5c471..42202636a 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -232,7 +232,7 @@ steps: prompt: "Implement issue #{{ inputs.issue }}." ``` -A `deliver: pr` step that would run read-only fails validation. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. +A `deliver: pr` step that would run read-only fails validation, and so does a delivering `workflow` step, `for_each` or not, when no step of its workflow writes a branch. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. A run that a crash stopped, or that another machine took over, finds the pull request it opened before and opens no other. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 3e1c8a1bd..f4371d824 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -232,7 +232,7 @@ steps: prompt: "Implement issue #{{ inputs.issue }}." ``` -会以只读方式运行的 `deliver: pr` 步骤无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 +会以只读方式运行的 `deliver: pr` 步骤无法通过校验；设置了 `deliver: pr` 的 `workflow` 步骤（无论是否 `for_each`），若其工作流中没有步骤写分支，也无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。因崩溃而中断、或被另一台机器接管的运行，会找到之前开的 pull request，不会再开第二个。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 diff --git a/src/__tests__/workflow-engine.test.ts b/src/__tests__/workflow-engine.test.ts index a85aa4ff0..f7bdb96b1 100644 --- a/src/__tests__/workflow-engine.test.ts +++ b/src/__tests__/workflow-engine.test.ts @@ -4467,6 +4467,30 @@ describe('deliver: pr at workflow start', () => { } }); }); + it.each([ + ['refuses', '', { error: { + code: 'bad_delivery', + message: 'Step "each" has deliver: pr, but no step of the workflow it runs writes a branch, so it would deliver nothing. Set workspace: new on "each" so its prompt steps write, or give a step in that workflow workspace: new or continue.', + } }], + ['starts', ' workspace: new\n', { data: { status: 'running' } }], + ])('%s a for_each workflow step that delivers, by whether a step its items run writes a branch', async (_case, workspace, expected) => { + const { config } = testConfig(); + fs.writeFileSync(path.join(config.repo.localPath, 'workflows', 'review.yaml'), workflow('review', ` - id: read + prompt: Review the package. + - id: lint + needs: [read] + command: npm run lint +`)); + fs.writeFileSync(path.join(config.repo.localPath, 'workflows', 'ship.yaml'), workflow('ship', ` - id: each + for_each: inputs.packages +${workspace} deliver: pr + workflow: review +`, 'inputs: { packages: [string] }\n')); + expect((await syncWorkflowDefinitions(config)).invalid).toEqual([]); + + expect(await startRun(config, 'ship', { packages: ['web'] }, { base: { ref: 'main', commit } })).toMatchObject(expected); + }); + it('refuses a run that delivers from a detached HEAD, telling to start from a named branch', async () => { const { started } = await start({ ship: workflow('ship', ` - id: fix workspace: new diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 80cf36936..20b3cbf1a 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -88,19 +88,27 @@ export function deliveryProblem( definition: WorkflowDefinition, workflows: Record, base: RunState['base'], ): WorkflowFailure | undefined { let delivering: string | undefined; + const childOf = (step: WorkflowStep) => (typeof step.workflow === 'string' ? workflows[step.workflow] : step.workflow); + const writes = (steps: WorkflowStep[], ancestors: WorkflowStep[]): boolean => steps.some((step) => { + const child = childOf(step); + return child ? writes(child.steps, [...ancestors, step]) : resolveMode(step, ancestors) !== 'read-only'; + }); const visit = (steps: WorkflowStep[], ancestors: WorkflowStep[], prefix: string): WorkflowFailure | undefined => { for (const step of steps) { const path = `${prefix}${step.id}`; + const child = childOf(step); if (step.deliver === 'pr') { delivering ??= path; if (step.workflow !== undefined && step.workspace !== 'new' && !step.forEach) { return { code: 'bad_delivery', message: `Step "${path}" delivers its workflow, so it needs workspace: new to own the branch it delivers.` }; } + if (child && !writes(child.steps, [...ancestors, step])) { + return { code: 'bad_delivery', message: `Step "${path}" has deliver: pr, but no step of the workflow it runs writes a branch, so it would deliver nothing. Set workspace: new on "${path}" so its prompt steps write, or give a step in that workflow workspace: new or continue.` }; + } if (step.workflow === undefined && resolveMode(step, ancestors) === 'read-only') { return { code: 'bad_delivery', message: `Step "${path}" has deliver: pr but runs read-only. Set workspace: new on it, or on the workflow step that runs it.` }; } } - const child = typeof step.workflow === 'string' ? workflows[step.workflow] : step.workflow; const problem = child && visit(child.steps, [...ancestors, step], `${path}.`); if (problem) return problem; } From 1247f9e5584c5d91004f47021e820c5ca7b1f8e5 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:33:28 +0200 Subject: [PATCH 36/77] docs(workflow): say when a run another machine took over can open a second pull request The guide said a run another machine took over finds the pull request it opened before and opens no other. The machine that handed the run over may still be opening one at that moment, and only hosts that reject a duplicate request for the same branches keep that to one. Both guides now say so, and name GitHub and GitLab as such hosts. Red: no test; a docs-only change. The unqualified sentence matched once in each of docs/guide/workflow.md and docs/guide/zh-CN/workflow.md before this change, and in neither after it. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 42202636a..cf31195d9 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -234,7 +234,7 @@ steps: A `deliver: pr` step that would run read-only fails validation, and so does a delivering `workflow` step, `for_each` or not, when no step of its workflow writes a branch. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. -When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. A run that a crash stopped, or that another machine took over, finds the pull request it opened before and opens no other. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. +When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. A run that a crash stopped finds the pull request it opened before and opens no other. So does a run another machine took over, unless the machine that handed it over is still opening one at that moment. GitHub and GitLab reject such a duplicate. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. A pull request goes into the nearest branch its own branch grew from that already delivered one, or else into the branch the run started from. A branch grows first from its `for_each` item's `after` branches, then from its `needs` in order. The first step of a sub-workflow grows from the needs of the step that runs it. A skipped step wrote no branch, so it is passed over. Each pull request then shows only the work done since the branch it goes into. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index f4371d824..d71fa5e21 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -234,7 +234,7 @@ steps: 会以只读方式运行的 `deliver: pr` 步骤无法通过校验；设置了 `deliver: pr` 的 `workflow` 步骤（无论是否 `for_each`），若其工作流中没有步骤写分支，也无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 -步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。因崩溃而中断、或被另一台机器接管的运行，会找到之前开的 pull request，不会再开第二个。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 +步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。因崩溃而中断的运行，会找到之前开的 pull request，不会再开第二个。被另一台机器接管的运行也是如此，除非交出运行的机器此刻仍在开 pull request；GitHub 和 GitLab 会拒绝这样的重复请求。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 pull request 合入这条分支来源中最近一条已经交付过的分支，没有时合入运行开始时的分支。分支首先来自其 `for_each` item 的 `after` 分支，然后依次来自 `needs`。子工作流的第一个步骤来自运行它的步骤的 needs。跳过的步骤没有写分支，会被越过。因此每个 pull request 只显示它所合入的分支之后的工作。 From 67e88047a6791658653d8c43d3911fadd754ff0a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:36:19 +0200 Subject: [PATCH 37/77] fix(github): look up an open pull request without blocking the process The GitHub lookup ran gh through ghExec, a synchronous spawn. The drain runs in every workflow next, and the local driver calls it in the process that also reads its agents' output, runs their timeouts and handles signals, so a slow gh froze all of that for up to the lookup timeout. The lookup now runs gh through cross-spawn's asynchronous form, with the same arguments, environment and timeout as ghExec. The GitLab, TGit and GitCode lookups already use fetch, and CNB has none. The create paths are unchanged and still spawn synchronously. Red: "keeps the process running while gh answers, and gives up on a gh that does not answer in time" failed with `gh could not look up open pull requests. Check gh auth status and its access to the repository.`: the timer that lets the fake gh answer could not fire while gh ran, so the lookup timed out after 5 seconds. --- src/__tests__/workflow-pr-providers.test.ts | 15 +++++++++++ src/providers/github/gh-cli.ts | 28 +++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 464020092..5ff413a5b 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -52,6 +52,21 @@ describe('GitHub pull request lookup', () => { expect(gh.calls()).toMatchObject([{ args: ['api', expect.stringMatching(/^repos\/acme\/app\/pulls\?/)], prompt: '0' }]); }); + it('keeps the process running while gh answers, and gives up on a gh that does not answer in time', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-fake-gh-')); + dirs.push(dir); + fakeGh(dir, { hold: 'lookup', open: [pull(ref.source)] }); + await expect(lookup().findOpen({ ...ref, timeoutMs: 300 })).rejects.toThrow('gh could not look up open pull requests'); + + // Only a timer of this process lets gh answer, so a lookup that blocked the process would time out instead. + const release = setTimeout(() => fs.writeFileSync(path.join(dir, 'release'), ''), 50); + try { + expect(await lookup().findOpen({ ...ref, timeoutMs: 5000 })).toBe('https://github.com/acme/app/pull/7'); + } finally { + clearTimeout(release); + } + }); + it('fails, rather than finding nothing, without credentials or on an answer it cannot read, and never repeats what GitHub said', async () => { resolveGh(null); vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index f4f4d8ae2..baad8b2e9 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -628,8 +628,9 @@ export async function ghFindOpenPull({ repo, source, target, timeoutMs }: PullRe } async function ghApiGet(path: string, timeoutMs: number): Promise { - if (isGhInstalled()) { - const result = ghExec(['api', path], { timeoutMs, env: { GH_PROMPT_DISABLED: '1', GIT_TERMINAL_PROMPT: '0' } }); + const ghPath = getGhPath(); + if (ghPath) { + const result = await ghSpawn(ghPath, ['api', path], { timeoutMs, env: { GH_PROMPT_DISABLED: '1', GIT_TERMINAL_PROMPT: '0' } }); if (result.status !== 0) throw new Error('gh could not look up open pull requests. Check gh auth status and its access to the repository.'); try { return JSON.parse(result.stdout); @@ -651,3 +652,26 @@ async function ghApiGet(path: string, timeoutMs: number): Promise { throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); } } + +/** + * Runs gh as `ghExec` does, through cross-spawn with the same arguments, environment and timeout, but without blocking + * the process, which a workflow driver shares with the agents it runs, while gh waits on GitHub. + */ +function ghSpawn(ghPath: string, args: string[], { env, timeoutMs }: { env: NodeJS.ProcessEnv; timeoutMs: number }): Promise<{ stdout: string; status: number }> { + log.debug(`gh exec: ${ghPath} ${args.join(' ')}`); + return new Promise((resolve) => { + const child = crossSpawn(ghPath, args, { env: { ...process.env, ...env }, timeout: timeoutMs }); + let stdout = ''; + let settled = false; + const finish = (status: number): void => { + if (settled) return; + settled = true; + resolve({ stdout: stdout.trim(), status }); + }; + child.stdout?.setEncoding('utf8'); + child.stdout?.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr?.resume(); + child.on('error', () => finish(1)); + child.on('close', (code) => finish(code ?? 1)); + }); +} From a467b0b99afeb410d8ec2c2c78abe0fe56df0deb Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:04:50 +0200 Subject: [PATCH 38/77] fix(workflow): read changes.pr_url sent without the branch facts A driver whose commit and push teamai does fills no branch, base, from, to or commits itself, so the pull request it opened can only arrive as changes.pr_url alone. parseReport rejected that report, because it required the branch facts whenever changes was present. changes may now hold pr_url alone. A kept URL that no delivery takes is named in dropped with its reason, as for a report with the branch facts. Red: 'reads changes.pr_url sent alone, from a driver that leaves its other changes to teamai' failed with parseReport answering 'changes must include branch, base, from, and to'; 'drops a URL sent alone from a report that succeeds/fails' failed with no dropped entry in the envelope. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 19 +++++++++++++++++++ src/workflow/engine.ts | 20 +++++++++++++------- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index cf31195d9..a1ffc2308 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,7 +244,7 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. -A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url`, with the step's other `changes` if it reports them. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index d71fa5e21..b1b787f4c 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,7 +244,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 -自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url` 上报；如果它也上报步骤的其他 `changes`，就一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index b3d1b45d2..0d47d7538 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -162,6 +162,12 @@ describe('workflow PR delivery contract', () => { expect(JSON.stringify(parsed)).not.toContain('secret'); }); + it('reads changes.pr_url sent alone, from a driver that leaves its other changes to teamai', () => { + const driver = { kind: 'remote', machine: 'host', workspace: 'work' }; + expect(parseReport({ attempt: 'a1', revision: 1, outcome: 'success', driver, changes: { pr_url: 'https://review.example/1' } })) + .toEqual({ report: { attempt: 'a1', revision: 1, outcome: 'success', driver, prUrl: 'https://review.example/1' }, ignored: [] }); + }); + it.each([ ['https://github.com/acme/app/pull/101', { url: 'https://github.com/acme/app/pull/101' }], ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', { url: 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview' }], @@ -738,6 +744,19 @@ describe('reported URLs from a driver', () => { expect(gh.calls()).toEqual([]); }); + it.each([ + ['succeeds with no commits', 'success', 'succeeded', 'the branch has no changes beyond the run base'], + ['fails', 'failed', 'failed', 'the step failed'], + ])('drops a URL sent alone from a report that %s, saying why, since nothing delivers it', async (_case, outcome, status, reason) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + expect(await report(config, runId, { attempt: fix.attempt, revision: fix.revision, outcome, changes: { pr_url: 'https://github.com/fixture/app/pull/1' } })) + .toMatchObject({ data: { accepted: true, step: { id: 'fix', status }, dropped: [{ field: 'changes.pr_url', reason }] } }); + expect((await state(config, runId)).deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + it.each([ ['is not the open pull request of its branch', [7], (branch: string) => `is not the open pull request of branch ${branch} into main`, 7], ['names a branch with no open pull request', [], (branch: string) => `branch ${branch} has no open pull request into main`, 1], diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 558e792e5..507fc04e5 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1190,6 +1190,9 @@ const REPORT_FIELDS = new Set([ 'attempt', 'revision', 'outcome', 'driver', 'outputs', 'exitCode', 'message', 'tool', 'model', 'session', 'resumed', 'tokens', 'failure', 'changes', 'read', 'agent', 'used', ]); +/** The `changes` fields that say what a branch holds, which `parseReport` reads together. */ +const CHANGES_FACTS = ['branch', 'base', 'from', 'to', 'commits'] as const; + /** * The payload fields this CLI does not read: top-level names first, then `driver.` names. * Names only, never values, because an unknown field may hold anything (#1164). @@ -1283,13 +1286,16 @@ export function parseReport(value: unknown): ParsedReport | string { if (record.changes !== undefined) { if (!record.changes || typeof record.changes !== 'object' || Array.isArray(record.changes)) return 'changes must be an object'; const value = record.changes as Record; - if (typeof value.branch !== 'string' || typeof value.base !== 'string' || typeof value.from !== 'string' || typeof value.to !== 'string') { - return 'changes must include branch, base, from, and to'; - } - if (!Array.isArray(value.commits) || !value.commits.every((commit): commit is string => typeof commit === 'string')) { - return 'changes.commits must be an array of strings'; + // A driver whose commit and push teamai does, so it fills the branch facts, sends pr_url alone. + if (value.pr_url === undefined || CHANGES_FACTS.some((key) => value[key] !== undefined)) { + if (typeof value.branch !== 'string' || typeof value.base !== 'string' || typeof value.from !== 'string' || typeof value.to !== 'string') { + return 'changes must include branch, base, from, and to'; + } + if (!Array.isArray(value.commits) || !value.commits.every((commit): commit is string => typeof commit === 'string')) { + return 'changes.commits must be an array of strings'; + } + report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; } - report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; if (value.pr_url !== undefined) report.prUrl = value.pr_url; } if (record.read !== undefined) { @@ -1309,7 +1315,7 @@ export function parseReport(value: unknown): ParsedReport | string { } const ignored = unreadFields(record, REPORT_FIELDS); if (isRecord(record.changes)) { - const known = new Set(['branch', 'base', 'from', 'to', 'commits', 'pr_url']); + const known = new Set([...CHANGES_FACTS, 'pr_url']); ignored.push(...Object.keys(record.changes).filter((key) => !known.has(key)).map((key) => `changes.${key}`)); } return { report, ignored }; From 90363fbc3515ad26ade54100f2159169b686dad6 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 17:47:31 +0200 Subject: [PATCH 39/77] docs(workflow): say where a condition that stops a run must live for delivery to honour it The gate before a pull request is opened reconciles the run under its lock, so a stop condition in reconcileRun opens nothing once it holds. A check made only later in next would let the drain open a pull request first. --- docs/dev/workflow-engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index c8452ecf2..3929cb494 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, none removes it, failed or unsafe keeps it. `unresolvedCreate` picks each once per drain, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. From 3fa365f0cb6d41e227e5858b8731c22b13afb9ea Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:05:08 +0200 Subject: [PATCH 40/77] fix(workflow): keep cancelled writes out of automatic delivery Keep the completed write revision for explicit recovery, records and scoring. Select only eligible children when delivering a workflow or summarizing its item branches. Red: winning child and explicit cancellation recovery tests failed with the cancelled branch selected and the older shared-branch commit used. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 74 ++++++++++++++++++++++++- src/workflow/fan-out.ts | 1 + src/workflow/step-state.ts | 2 +- src/workflow/workspace.ts | 2 +- 5 files changed, 77 insertions(+), 4 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 3929cb494..fd8cea4d1 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -9,7 +9,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A run ends only in `settleRun`.** The shared `finishRun` path calls it when settling reaches the top workflow's result, before it hands any lease back or out, and on a call that takes no lease too. `reportAttempt` first reconciles earlier reports, so an end cancels a later report before it can advance the revision. It records an accepted outcome and never ends the run. After a failed report, the next `next` skips the dependents and ends the run. Respond, cancel and retry in `run-controls.ts`, and next in `engine.ts`, share that path. Whole-run cancel settles directly as `cancelled`, without reconciling another round. To run code when a run ends, hook into `finishRun`. Do network I/O only after `withRun` releases the run lock. That path queues the end record, a local write, under the lock, and saves whether it did in the run's `endRecordQueued` field. When `updateRun` then fails to save the run file, the queue is one save ahead of the run file, and the next `next` settles the run again and overwrites the queued `.end.json`. A later `next` reports a record state only for a run whose field is true (`runRecordState` in `records.ts`). - **Reconcile the latest report before applying another or handing out a lease.** `reconcileRun`, shared by `leaseNext` and `reportAttempt`, settles every driverless transition to a fixpoint at the run's current revision, through `settleScope` and `settleItems`, including workflow starts, fan-out expansion, render failures and bound exhaustion. `reportAttempt` advances the revision only after it, so only the latest revision can be unreconciled. A reached end applies before a loop decision. Only after reconciliation does `leaseNext` hand back an omitted open lease or issue the first ready leaf in declaration order that this driver can take. A running workflow occupies its parent's position in that order. Settling returns every ready leaf in that order (`ready`), and `leaseNext` walks it. A ready item does not count against `max_parallel` until it is leased, and one `next` leases at most one step, so a held-back item leaves its room to the next one without exceeding the cap. `canTakeLease: false` still reconciles driverless transitions. Every changed run is saved, including a `wait`. - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. -- **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. +- **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. A cancelled delivering step keeps its agent settlement and its original `settledAt` for explicit recovery, records and scoring. Automatic delivery and workflow-item branch summaries exclude cancelled children. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 0d47d7538..b6cfad5a1 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -73,7 +73,7 @@ async function state(config: LocalConfig, runId: string): Promise { const LeaseSchema = z.object({ step: z.string(), attempt: z.string(), revision: z.number(), pass: z.number(), - workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), + workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }), start: z.object({ commit: z.string() }).optional() }).optional(), }); type Lease = z.infer; const remote = { kind: 'remote', machine: 'paseo', workspace: 'session-1' }; @@ -802,6 +802,78 @@ function providerDouble(findOpen: PullRequestLookup['findOpen'], create: GitProv } describe('retrying and cancelling a delivery', () => { + it('delivers the winning child branch after an end cancels a sibling delivery', async () => { + const { config, root, runId } = await fixture(` - id: ship + for_each: inputs.packages + deliver: pr + workflow: + ends: + done: succeeded + steps: + - id: loser + command: echo loser + workspace: new + deliver: pr + - id: winner + needs: [] + command: echo winner + workspace: new + end: done +`, { packages: [{ name: 'a', id: 'a', after: [] }] }); + const gh = fakeGh(path.join(root, 'gh')); + config.workflows = { tools: {}, ...config.workflows, max_parallel: 2 }; + const loser = await take(config, runId); + const winnerNext = await leaseNext(config, runId, remote, { ...writable, running: [loser.attempt] }); + if ('error' in winnerNext) throw new Error(winnerNext.error.message); + if (!winnerNext.data.lease) throw new Error(JSON.stringify(winnerNext)); + const winner = LeaseSchema.parse(winnerNext.data.lease); + await succeed(config, runId, loser, { to: pushWork(root, loser, { 'loser.txt': 'loser' }) }); + await report(config, runId, { attempt: winner.attempt, revision: winner.revision, outcome: 'success', + changes: { branch: winner.workspace?.branch, base: winner.workspace?.base.commit, from: winner.workspace?.base.commit, + to: pushWork(root, winner, { 'winner.txt': 'winner' }), commits: [] } }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'ship[0].loser')).toMatchObject({ status: 'cancelled', settledAt: 2 }); + expect(creates(gh)).toEqual([[winner.workspace?.branch, 'main']]); + expect(run.steps.find((step) => step.id === 'ship')).toMatchObject({ results: [{ branch: winner.workspace?.branch }] }); + }); + + it('starts explicit cancellation recovery from the latest recorded write on a shared branch', async () => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue + - id: recover + needs: [impl, docs] + if: steps.impl.outcome == 'cancelled' + command: echo recover + workspace: new +`); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const to = pushOnto(root, docs.workspace?.branch ?? '', { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to }); + const settledAt = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + await cancelRun(config, runId, 'work.impl'); + + const recovery = await take(config, runId); + expect(recovery).toMatchObject({ step: 'work.recover', workspace: { start: { commit: to } } }); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ settledAt: settledAt && 'settledAt' in settledAt ? settledAt.settledAt : undefined }); + }); + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); diff --git a/src/workflow/fan-out.ts b/src/workflow/fan-out.ts index 23701ae0e..4c31ccec5 100644 --- a/src/workflow/fan-out.ts +++ b/src/workflow/fan-out.ts @@ -127,6 +127,7 @@ export function itemsSettled( function branchesFromChildren(run: RunState): Map { const best = new Map(); for (const step of run.steps) { + if (step.status === 'cancelled') continue; const branch = 'changes' in step ? step.changes?.branch : undefined; if (!branch) continue; const settledAt = 'settledAt' in step ? step.settledAt : 0; diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 08d241cdf..37fd27f03 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -329,7 +329,7 @@ export function fail(state: StepState, settlement: Settlement): Fail export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { if (state.status === 'delivering') { const { id: _id, status: _status, ...settlement } = state; - return become(state, { ...settlement, status: 'cancelled', settledAt, failure }); + return become(state, { ...settlement, status: 'cancelled', failure }); } const attempt = state.status === 'leased' ? state.attempt : undefined; return become(state, { status: 'cancelled', settledAt, ...(attempt !== undefined ? { attempt } : {}), failure }); diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 20b3cbf1a..172c23f21 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -168,7 +168,7 @@ export function deliverableOf(run: RunState, state: StepState, step: WorkflowSte let head = changes?.to; if (step.workflow !== undefined) { const own = branchName(run.id, state.id, state.pass ?? 1); - const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && changesOf(item)) + const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status !== 'cancelled' && changesOf(item)) .sort((left, right) => ('settledAt' in right ? right.settledAt : 0) - ('settledAt' in left ? left.settledAt : 0))[0]; branch = run.copies?.[own]?.commit !== undefined ? own : changesOf(latest)?.branch; head = branch === undefined ? undefined : run.copies?.[branch]?.commit; From 9c65c3922b387687741392654b7790ef02f8037c Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:06:27 +0200 Subject: [PATCH 41/77] fix(workflow): deliver later branch tips recorded by reports Read the recorded branch tip under the run lock. A retry may deliver that tip after checking its diff against the run base; unrecorded tips still fail. Red: later recorded shared-branch tip tests failed with a clean delivery failing and a protected tip classified as branch_moved. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 39 +++++++++++++++++++++++++ src/workflow/engine.ts | 1 + src/workflow/pr-delivery.ts | 10 ++++++- 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index fd8cea4d1..112b4c091 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, none removes it, failed or unsafe keeps it. `unresolvedCreate` picks each once per drain, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index b6cfad5a1..610bf4e19 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -874,6 +874,45 @@ describe('retrying and cancelling a delivery', () => { expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ settledAt: settledAt && 'settledAt' in settledAt ? settledAt.settledAt : undefined }); }); + it.each([false, true])('retries delivery at a later recorded shared-branch tip and checks its protected paths (%s)', async (protectedChange) => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const docs = await take(config, runId); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed' } }); + const file = protectedChange ? '.github/workflows/docs.yml' : 'docs.txt'; + const to = pushOnto(root, docs.workspace?.branch ?? '', { [file]: 'docs' }); + await succeed(config, runId, docs, { to }); + expect((await state(config, runId)).copies?.[docs.workspace?.branch ?? '']?.commit).toBe(to); + gh.configure({}); + await retryStep(config, runId, 'work.impl'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: protectedChange ? 'failed' : 'succeeded' } } }); + const run = await state(config, runId); + if (protectedChange) { + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining(file) } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + } else { + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + expect(creates(gh)).toEqual([[docs.workspace?.branch, 'main']]); + expect(run.deliveries).toHaveLength(1); + } + }); + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 507fc04e5..7edd24a73 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -915,6 +915,7 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { } return { key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, + recordedHead: run.copies?.[key.branch]?.commit, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index ab9aaab00..07ca4b9be 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -34,6 +34,8 @@ export interface DeliveryJob { checkout: string; /** The run base commit the delivered branch is compared with. */ base: string; + /** The latest commit a report recorded on this branch, read under the run lock. */ + recordedHead?: string; /** The branch the pull request goes into. */ target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ @@ -86,7 +88,13 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise 0) return failed({ kind: 'protected_path', files: protectedFiles }); + } if (job.reported) return { ok: true, pr: job.reported }; const origin = await originOf(job.checkout); if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); From 69a75d39e57e80207682fae5b299af23a928f1ba Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:06:56 +0200 Subject: [PATCH 42/77] Revert "fix(workflow): read changes.pr_url sent without the branch facts" This reverts commit a467b0b99afeb410d8ec2c2c78abe0fe56df0deb. No path in this change fills branch facts for a URL-only report, so accepting one loses the pull request. The old refusal tells the driver what to send. Red: refuses a reported pull request without branch facts failed with parseReport accepting the URL-only report. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 17 ++--------------- src/workflow/engine.ts | 20 +++++++------------- 4 files changed, 11 insertions(+), 30 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index a1ffc2308..cf31195d9 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,7 +244,7 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. -A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url`, with the step's other `changes` if it reports them. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index b1b787f4c..d71fa5e21 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,7 +244,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 -自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url` 上报；如果它也上报步骤的其他 `changes`，就一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 610bf4e19..ceac58f42 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -162,10 +162,10 @@ describe('workflow PR delivery contract', () => { expect(JSON.stringify(parsed)).not.toContain('secret'); }); - it('reads changes.pr_url sent alone, from a driver that leaves its other changes to teamai', () => { + it('refuses a reported pull request without branch facts', () => { const driver = { kind: 'remote', machine: 'host', workspace: 'work' }; expect(parseReport({ attempt: 'a1', revision: 1, outcome: 'success', driver, changes: { pr_url: 'https://review.example/1' } })) - .toEqual({ report: { attempt: 'a1', revision: 1, outcome: 'success', driver, prUrl: 'https://review.example/1' }, ignored: [] }); + .toBe('changes must include branch, base, from, and to'); }); it.each([ @@ -744,19 +744,6 @@ describe('reported URLs from a driver', () => { expect(gh.calls()).toEqual([]); }); - it.each([ - ['succeeds with no commits', 'success', 'succeeded', 'the branch has no changes beyond the run base'], - ['fails', 'failed', 'failed', 'the step failed'], - ])('drops a URL sent alone from a report that %s, saying why, since nothing delivers it', async (_case, outcome, status, reason) => { - const { config, root, runId } = await fixture(FIX_THEN_REVIEW); - const gh = fakeGh(path.join(root, 'gh')); - const fix = await take(config, runId); - expect(await report(config, runId, { attempt: fix.attempt, revision: fix.revision, outcome, changes: { pr_url: 'https://github.com/fixture/app/pull/1' } })) - .toMatchObject({ data: { accepted: true, step: { id: 'fix', status }, dropped: [{ field: 'changes.pr_url', reason }] } }); - expect((await state(config, runId)).deliveries).toBeUndefined(); - expect(gh.calls()).toEqual([]); - }); - it.each([ ['is not the open pull request of its branch', [7], (branch: string) => `is not the open pull request of branch ${branch} into main`, 7], ['names a branch with no open pull request', [], (branch: string) => `branch ${branch} has no open pull request into main`, 1], diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 7edd24a73..ec84ba93d 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -1191,9 +1191,6 @@ const REPORT_FIELDS = new Set([ 'attempt', 'revision', 'outcome', 'driver', 'outputs', 'exitCode', 'message', 'tool', 'model', 'session', 'resumed', 'tokens', 'failure', 'changes', 'read', 'agent', 'used', ]); -/** The `changes` fields that say what a branch holds, which `parseReport` reads together. */ -const CHANGES_FACTS = ['branch', 'base', 'from', 'to', 'commits'] as const; - /** * The payload fields this CLI does not read: top-level names first, then `driver.` names. * Names only, never values, because an unknown field may hold anything (#1164). @@ -1287,16 +1284,13 @@ export function parseReport(value: unknown): ParsedReport | string { if (record.changes !== undefined) { if (!record.changes || typeof record.changes !== 'object' || Array.isArray(record.changes)) return 'changes must be an object'; const value = record.changes as Record; - // A driver whose commit and push teamai does, so it fills the branch facts, sends pr_url alone. - if (value.pr_url === undefined || CHANGES_FACTS.some((key) => value[key] !== undefined)) { - if (typeof value.branch !== 'string' || typeof value.base !== 'string' || typeof value.from !== 'string' || typeof value.to !== 'string') { - return 'changes must include branch, base, from, and to'; - } - if (!Array.isArray(value.commits) || !value.commits.every((commit): commit is string => typeof commit === 'string')) { - return 'changes.commits must be an array of strings'; - } - report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; + if (typeof value.branch !== 'string' || typeof value.base !== 'string' || typeof value.from !== 'string' || typeof value.to !== 'string') { + return 'changes must include branch, base, from, and to'; + } + if (!Array.isArray(value.commits) || !value.commits.every((commit): commit is string => typeof commit === 'string')) { + return 'changes.commits must be an array of strings'; } + report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; if (value.pr_url !== undefined) report.prUrl = value.pr_url; } if (record.read !== undefined) { @@ -1316,7 +1310,7 @@ export function parseReport(value: unknown): ParsedReport | string { } const ignored = unreadFields(record, REPORT_FIELDS); if (isRecord(record.changes)) { - const known = new Set([...CHANGES_FACTS, 'pr_url']); + const known = new Set(['branch', 'base', 'from', 'to', 'commits', 'pr_url']); ignored.push(...Object.keys(record.changes).filter((key) => !known.has(key)).map((key) => `changes.${key}`)); } return { report, ignored }; From 05e001a6b4da87cfaf91f8418e2b883d8c5690b3 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:09:25 +0200 Subject: [PATCH 43/77] fix(workflow): bound unresolved create reconciliation across drains Count lookup outcomes, defer new or failed creates until a later drain, and drop unresolved entries on the second empty, third failed, or first unsafe answer. Publish the counters with the run. Red: unresolved-create policy tests failed with three lookups in the create drain and no marker left for a delayed receipt. --- docs/dev/workflow-engine.md | 4 +- src/__tests__/workflow-delivery.test.ts | 67 ++++++++++++++++++++++++- src/workflow/engine.ts | 24 ++++++--- src/workflow/pr-delivery.ts | 9 ++-- src/workflow/snapshot.ts | 3 +- 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 112b4c091..f0b1fad83 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -14,9 +14,9 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. - **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. -- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, none removes it, failed or unsafe keeps it. `unresolvedCreate` picks each once per drain, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. +- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. -- **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. +- **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. - **A step waits for every step it reads.** `readsOf` collects the steps that the step's expressions (`stepExpressions`), templates, and `with` read. For a step that runs an inline workflow, it also collects the outer steps that the inline steps read. `skipUnreachable` uses that set to decide readiness. If a new field reads steps, add it to `stepExpressions` or `templatedTexts`. Otherwise the step can read a value before it exists, or an inline workflow can wait forever. A step outside a loop that needs or reads a step inside it also waits until the loop owner is settled (`waitsFor` in `loop.ts`), and is neither skipped nor ready before. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index ceac58f42..91e6618b5 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1012,6 +1012,65 @@ describe('retrying and cancelling a delivery', () => { expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); }); + it.each<[string, number]>([ + ['none', 2], ['failed', 3], ['unsafe', 1], + ])('bounds unresolved-create lookups answering %s across drains', async (answer, limit) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let recovering = false; + const lookup = vi.fn(async () => { + if (!recovering) return undefined; + if (answer === 'failed') throw new Error('host unavailable'); + return answer === 'unsafe' ? 'https://other.example/pull/7' : undefined; + }); + const create = providerDouble(lookup, async () => { + await cancelRun(config, runId); + throw new Error('create timed out'); + }); + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(2); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 0, none: 0, failed: 0 } })]); + recovering = true; + lookup.mockClear(); + + for (let count = 1; count <= limit; count += 1) { + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(count); + const run = await state(config, runId); + if (count < limit) { + expect(run.creating).toEqual([expect.objectContaining({ lookups: { total: count, none: answer === 'none' ? count : 0, failed: answer === 'failed' ? count : 0 } })]); + const projected = projectRun(run, {}); + expect(projected.ok && projected.value.creating).toEqual(run.creating); + } else expect(run.creating).toBeUndefined(); + expect(run.deliveries).toBeUndefined(); + } + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(limit); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('keeps an empty unresolved lookup until a later drain finds its receipt', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let url: string | undefined; + const lookup = vi.fn(async () => url); + providerDouble(lookup, async () => { + await cancelRun(config, runId); + throw new Error('create timed out'); + }); + await leaseNext(config, runId, remote, writable); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + url = 'https://github.com/fixture/app/pull/7'; + await leaseNext(config, runId, remote, writable); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(lookup).toHaveBeenCalledTimes(4); + }); + it('records a pull request a cancelled delivery opened before its process stopped, by looking it up on a later call, and opens no other', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); config.usageReportEnabled = true; @@ -1072,7 +1131,9 @@ describe('retrying and cancelling a delivery', () => { expect(create).toHaveBeenCalledTimes(1); expect(run.deliveries).toBeUndefined(); expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); - expect(run.creating).toBeUndefined(); + expect(run.creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toBeUndefined(); }); it('records a pull request a cancelled delivery opened before its process stopped, when the step is retried before any later call', async () => { @@ -1122,6 +1183,8 @@ describe('retrying and cancelling a delivery', () => { return url; }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect(retried).toBe(false); expect(await take(config, runId)).toMatchObject({ step: 'review' }); const run = await state(config, runId); expect(retried).toBe(true); @@ -1159,7 +1222,7 @@ describe('retrying and cancelling a delivery', () => { fs.rmSync(deliveryLockPath(config, runId), { force: true }); const marked = await state(config, runId); expect(marked.steps[0]).toMatchObject({ id: 'fix', status: 'delivering' }); - expect(marked.creating).toEqual([{ step: 'fix', pass: 1, settledAt: 2, branch, head: to, target: 'main' }]); + expect(marked.creating).toEqual([{ step: 'fix', pass: 1, settledAt: 2, branch, head: to, target: 'main', lookups: { total: 0, none: 0, failed: 0 } }]); const handed = projectRun(marked, {}); expect(handed.ok && handed.value.creating).toEqual(marked.creating); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index ec84ba93d..b11e33751 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -923,18 +923,18 @@ function nextDelivery(run: RunState): DeliveryJob | undefined { } /** The first request in `run.creating` that this drain has not looked up yet. */ -function unresolvedCreate(run: RunState, tried: readonly UnresolvedCreate[]): { create: UnresolvedCreate; checkout: string } | undefined { +function unresolvedCreate(run: RunState, tried: readonly Omit[]): { create: UnresolvedCreate; checkout: string } | undefined { const { checkout } = run; const create = run.creating?.find((listed) => !tried.some((done) => sameCreate(done, listed))); return create && checkout ? { create, checkout } : undefined; } -function sameCreate(a: UnresolvedCreate, b: UnresolvedCreate): boolean { +function sameCreate(a: Omit, b: Omit): boolean { return sameKey(a, b) && a.target === b.target; } /** Drops `create` from the requests the run has no receipt for. */ -function forgetCreate(run: RunState, create: UnresolvedCreate): void { +function forgetCreate(run: RunState, create: Omit): void { const left = run.creating?.filter((listed) => !sameCreate(listed, create)); if (left?.length) run.creating = left; else delete run.creating; @@ -966,12 +966,19 @@ function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutc /** * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, - * and one it does not hold is dropped. One it could not answer for stays for a later drain. + * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. */ function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited>): { requeueEnd: boolean } { - if (found.kind === 'failed') return { requeueEnd: false }; + const listed = run.creating?.find((item) => sameCreate(item, create)); + if (!listed) return { requeueEnd: false }; + const lookups = listed.lookups ?? { total: 0, none: 0, failed: 0 }; + listed.lookups = { + total: lookups.total + 1, + none: lookups.none + (found.kind === 'none' ? 1 : 0), + failed: lookups.failed + (found.kind === 'failed' ? 1 : 0), + }; const receipted = found.kind === 'found' && recordReceipt(run, create, create.target, found.pr); - forgetCreate(run, create); + if (found.kind === 'found' || found.kind === 'unsafe' || listed.lookups.none >= 2 || listed.lookups.failed >= 3) forgetCreate(run, create); return { requeueEnd: receipted && run.status !== 'running' }; } @@ -987,7 +994,7 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); if (!step) return { value: 'refused' }; const create = { ...job.key, target: job.target }; - if (!run.creating?.some((listed) => sameCreate(listed, create))) (run.creating ??= []).push(create); + if (!run.creating?.some((listed) => sameCreate(listed, create))) (run.creating ??= []).push({ ...create, lookups: { total: 0, none: 0, failed: 0 } }); return { value: 'open', write: { ...run, updatedAt: new Date().toISOString() } }; }); return marked.status === 'done' ? marked.value : 'busy'; @@ -1007,7 +1014,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise { let requeued = false; - const tried: UnresolvedCreate[] = []; + const tried: Array> = []; for (;;) { const picked = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; @@ -1020,6 +1027,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise { requeueEnd: boolean }; if ('job' in picked.value) { const { job } = picked.value; + tried.push({ ...job.key, target: job.target }); const outcome = await deliverPr(job, () => markCreating(config, runId, job)); // As when the pick is busy: the step still delivers, and a later call delivers it. if (outcome === 'busy') return requeued; diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 07ca4b9be..33b78a85f 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -23,7 +23,7 @@ export interface DeliveryKey { * A pull request TeamAI asked a host to open, into `target`, and holds no receipt for yet. The run lists it until a * receipt or a lookup settles it, whatever becomes of its step, since the host may hold it. */ -export type UnresolvedCreate = DeliveryKey & { target: string }; +export type UnresolvedCreate = DeliveryKey & { target: string; lookups: { total: number; none: number; failed: number } }; /** Everything a delivery needs, read from the run under its lock. */ export interface DeliveryJob { @@ -148,15 +148,14 @@ export async function reportedPrMismatch( /** * Whether the host holds the open request `create` asked for, by lookup alone: nothing of the branch is read, and - * nothing is opened. `failed` is a host that cannot answer, an origin no provider reads, or a URL that cannot be kept. + * nothing is opened. `failed` is a host that cannot answer or an origin no provider reads. `unsafe` is a URL that cannot be kept. */ export async function findDelivered( checkout: string, { branch, target }: Pick, -): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' }> { +): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' } | { kind: 'unsafe' }> { const origin = await originOf(checkout); if (origin.kind === 'transport') return { kind: 'failed' }; - const found = await findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); - return found.kind === 'unsafe' ? { kind: 'failed' } : found; + return findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); } /** The provider of the checkout's configured origin; git alone when the checkout has none. */ diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 0c4b23332..ad186d119 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -38,8 +38,9 @@ function withCommitIds(schema: T): z.ZodEffects { }); } -const UnresolvedCreateSchema: z.ZodType = z.object({ +const UnresolvedCreateSchema: z.ZodType = z.object({ step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: z.string().min(1), target: z.string().min(1), + lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), }); const BodySchema = z.object({ From c115b94039bcc7c72385f16b1e642e905aa7312a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:11:08 +0200 Subject: [PATCH 44/77] fix(workflow): keep unconfirmed creates pending for canonical lookup A failed post-create lookup records no URL and fails delivery with a retry command. Keep the unresolved create so retry looks up its canonical receipt before creating again. Red: unconfirmed TGit-style create test failed with the alternate create URL recorded and the dependent leased. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 + docs/guide/zh-CN/workflow.md | 2 + src/__tests__/workflow-delivery.test.ts | 49 +++++++++++++++++++------ src/workflow/pr-delivery.ts | 7 ++++ 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index f0b1fad83..e476f76b3 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it, and treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index cf31195d9..7a2f4fbf4 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,6 +244,8 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. +If a host may have opened the request but TeamAI cannot confirm it, the step fails with `delivery_failed` and records no URL. Run the retry command in the message. TeamAI looks for the request before opening one again. + A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index d71fa5e21..87db49759 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,6 +244,8 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 +如果主机可能已打开请求，但 TeamAI 无法确认，步骤会以 `delivery_failed` 失败，且不记录 URL。运行消息中的重试命令。TeamAI 会先查找已有请求，再尝试打开请求。 + 自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 91e6618b5..6735d3954 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -637,24 +637,19 @@ describe('where a pull request goes', () => { }); it.each([ - ['lists a request on another host', async (): Promise => 'https://other.example/review/1', 'https://github.com/fixture/app/pull/1', false], + ['lists a request on another host', async (): Promise => 'https://other.example/review/1', 'https://github.com/fixture/app/pull/1', true], ['cannot answer, and the create named a request on another host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://other.example/review/1', false], - ['cannot answer, and the create named a request on the origin host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://github.com/fixture/app/pull/1', true], - ])('records the URL a create answered only on the origin host, when the lookup after it %s', async (_case, lookupAfter, answered, kept) => { + ['cannot answer, and the create named a request on the origin host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://github.com/fixture/app/pull/1', false], + ])('records no receipt, when the lookup after it %s', async (_case, lookupAfter, answered, unsafe) => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); const branch = `teamai/${runId}/fix/p1`; await succeed(config, runId, fix, { to: pushWork(root, fix) }); const create = providerDouble(async () => (create.mock.calls.length === 0 ? undefined : lookupAfter()), async () => answered); - if (kept) { - expect(await take(config, runId)).toMatchObject({ step: 'review' }); - expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual([answered]); - return; - } expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const run = await state(config, runId); - expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.`) } }); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(unsafe ? `answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.` : `may have opened a pull request from branch ${branch}, but TeamAI could not confirm it`) } }); expect(run.deliveries).toBeUndefined(); expect(create).toHaveBeenCalledTimes(1); }); @@ -781,10 +776,10 @@ describe('reported URLs from a driver', () => { * Puts a provider double on the fixture's GitHub origin: `findOpen` answers lookups and `create` opens requests. Returns * the create spy, so a test can show what was asked of the host. */ -function providerDouble(findOpen: PullRequestLookup['findOpen'], create: GitProvider['createPullRequest']) { +function providerDouble(findOpen: PullRequestLookup['findOpen'], create: GitProvider['createPullRequest'], hostname = 'github.com') { const provider = new GitHubProvider(); const creates = vi.spyOn(provider, 'createPullRequest').mockImplementation(create); - vi.spyOn(registry, 'providerForRemote').mockReturnValue({ kind: 'provider', provider, pullRequests: { findOpen }, repo: 'fixture/app', hostname: 'github.com' }); + vi.spyOn(registry, 'providerForRemote').mockReturnValue({ kind: 'provider', provider, pullRequests: { findOpen }, repo: 'fixture/app', hostname }); return creates; } @@ -900,6 +895,38 @@ describe('retrying and cancelling a delivery', () => { } }); + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const canonical = 'https://git.woa.com/fixture/app/merge_requests/7'; + let recovered = false; + const lookup = vi.fn(async () => { + if (recovered) return canonical; + if (create.mock.calls.length > 0) throw new Error('TGit lookup timed out'); + return undefined; + }); + const create = providerDouble(lookup, async () => 'https://git.woa.com/fixture/app/-/merge_requests/7', 'git.woa.com'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`may have opened a pull request from branch ${fix.workspace?.branch}, but TeamAI could not confirm it`) } }); + expect(failed.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(failed.deliveries).toBeUndefined(); + expect(failed.creating).toHaveLength(1); + expect(lookup).toHaveBeenCalledTimes(2); + + recovered = true; + await retryStep(config, runId, 'fix'); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([canonical]); + expect(buildEndRecord(run, remote, 'tester').prs).toEqual([canonical]); + expect(create).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(3); + }); + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 33b78a85f..2aef021d5 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -52,6 +52,7 @@ export type DeliveryFault = | { kind: 'no_provider' } | { kind: 'lookup_failed'; provider: string } | { kind: 'create_failed'; provider: string } + | { kind: 'create_unconfirmed'; provider: string } | { kind: 'unsafe_url'; provider: string } | { kind: 'withdrawn' }; @@ -124,6 +125,7 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 18:12:11 +0200 Subject: [PATCH 45/77] fix(github): settle lookup independently of launcher pipe closure Enforce the lookup deadline with a timer that settles the promise, kills the child and destroys stdio. A launcher descendant cannot keep delivery waiting on close. Red: launcher child deadline test failed with still waiting for the pipe after the lookup timeout. --- src/__tests__/workflow-pr-providers.test.ts | 29 +++++++++++++++++++++ src/providers/github/gh-cli.ts | 10 ++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts index 5ff413a5b..7930386f1 100644 --- a/src/__tests__/workflow-pr-providers.test.ts +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -67,6 +67,35 @@ describe('GitHub pull request lookup', () => { } }); + it('ends the lookup at its deadline when a launcher child keeps the output pipe open', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-gh-launcher-')); + dirs.push(dir); + const launcher = path.join(dir, 'gh'); + const pidFile = path.join(dir, 'holder.pid'); + fs.writeFileSync(launcher, `#!${process.execPath} +const { spawn } = require('node:child_process'); +const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'inherit' }); +require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(child.pid)); +setInterval(() => {}, 1000); +`, { mode: 0o755 }); + resolveGh(launcher); + let deadline: ReturnType | undefined; + const answer = lookup().findOpen({ ...ref, timeoutMs: 300 }).then(() => 'found', () => 'failed'); + try { + await vi.waitFor(() => expect(fs.existsSync(pidFile)).toBe(true)); + const result = await Promise.race([ + answer, + new Promise((resolve) => { deadline = setTimeout(() => resolve('still waiting for the pipe'), 1000); }), + ]); + expect(result).toBe('failed'); + } finally { + clearTimeout(deadline); + const pid = Number(fs.readFileSync(pidFile, 'utf8')); + process.kill(pid, 'SIGKILL'); + await answer; + } + }); + it('fails, rather than finding nothing, without credentials or on an answer it cannot read, and never repeats what GitHub said', async () => { resolveGh(null); vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index baad8b2e9..c929c11b4 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -660,14 +660,22 @@ async function ghApiGet(path: string, timeoutMs: number): Promise { function ghSpawn(ghPath: string, args: string[], { env, timeoutMs }: { env: NodeJS.ProcessEnv; timeoutMs: number }): Promise<{ stdout: string; status: number }> { log.debug(`gh exec: ${ghPath} ${args.join(' ')}`); return new Promise((resolve) => { - const child = crossSpawn(ghPath, args, { env: { ...process.env, ...env }, timeout: timeoutMs }); + const child = crossSpawn(ghPath, args, { env: { ...process.env, ...env } }); let stdout = ''; let settled = false; const finish = (status: number): void => { if (settled) return; settled = true; + clearTimeout(deadline); resolve({ stdout: stdout.trim(), status }); }; + const deadline = setTimeout(() => { + finish(1); + child.kill('SIGKILL'); + child.stdin?.destroy(); + child.stdout?.destroy(); + child.stderr?.destroy(); + }, timeoutMs); child.stdout?.setEncoding('utf8'); child.stdout?.on('data', (chunk: string) => { stdout += chunk; }); child.stderr?.resume(); From a52b6c9763c5a3fe998b455511ecc5bc527e79f2 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:13:11 +0200 Subject: [PATCH 46/77] fix(workflow): distinguish missing branches without credential prompts Report the missing origin branch and the commit to push before retrying. Suppress terminal and credential-manager prompts on the remote tip read. Red: missing origin branch test failed with the branch_moved recovery message. --- docs/guide/workflow.md | 2 ++ docs/guide/zh-CN/workflow.md | 2 ++ src/__tests__/workflow-delivery.test.ts | 20 ++++++++++++++++++++ src/workflow/pr-delivery.ts | 11 +++++++++-- src/workflow/worktree.ts | 4 ++-- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 7a2f4fbf4..cadfb9184 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,6 +244,8 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. +If origin has no branch to deliver, TeamAI opens no request. The failure names the missing branch and the commit to push to it, then gives the retry command. The origin check disables terminal and credential-manager prompts. + If a host may have opened the request but TeamAI cannot confirm it, the step fails with `delivery_failed` and records no URL. Run the retry command in the message. TeamAI looks for the request before opening one again. A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 87db49759..8d8380d70 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,6 +244,8 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 +如果 origin 没有要交付的分支，TeamAI 不会打开请求。失败消息会指出缺失的分支和要推送到该分支的提交，然后给出重试命令。检查 origin 时会禁用终端和凭据管理器的提示。 + 如果主机可能已打开请求，但 TeamAI 无法确认，步骤会以 `delivery_failed` 失败，且不记录 URL。运行消息中的重试命令。TeamAI 会先查找已有请求，再尝试打开请求。 自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 6735d3954..bc13c30bf 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -421,6 +421,26 @@ describe('delivering a step', () => { expect(creates(gh)).toEqual([]); }); + it('names a missing origin branch and reads it without credential prompts', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + const branch = fix.workspace?.branch ?? ''; + git(cwd, ['fetch', 'origin', branch]); + git(root, ['--git-dir=remote.git', 'update-ref', '-d', `refs/heads/${branch}`]); + await succeed(config, runId, fix, { to }); + vi.stubEnv('GIT_TERMINAL_PROMPT', '1'); + const reads = vi.spyOn(worktree, 'git'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`origin has no branch ${branch}; push ${to} to it, then run: teamai workflow retry ${runId} fix`) } }); + expect(reads).toHaveBeenCalledWith(cwd, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', `refs/heads/${branch}`], { timeoutMs: 30000, env: { GIT_TERMINAL_PROMPT: '0' } }); + expect(run.deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 2aef021d5..b583d0536 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -48,6 +48,7 @@ export type DeliveryFault = | { kind: 'unreadable' } | { kind: 'protected_path'; files: string[] } | { kind: 'branch_moved' } + | { kind: 'branch_missing' } | { kind: 'branch_unreadable' } | { kind: 'no_provider' } | { kind: 'lookup_failed'; provider: string } @@ -89,8 +90,9 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { const ref = `refs/heads/${branch}`; - const listed = await git(checkout, ['ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS }); + const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS, env: { GIT_TERMINAL_PROMPT: '0' } }); if (listed.code !== 0) return { kind: 'unreadable' }; const commit = listed.stdout.split('\n').map((line) => line.split('\t')).find(([, name]) => name?.trim() === ref)?.[0]; return commit ? { kind: 'read', commit } : { kind: 'read' }; @@ -228,6 +230,11 @@ export function deliveryFailure({ runId, key }: Pick { +export function git(cwd: string, args: string[], options?: { input: string } | { timeoutMs: number; env?: NodeJS.ProcessEnv }): Promise { if (options === undefined || !('input' in options)) { return new Promise((resolve) => { - execFile('git', args, { cwd, env: process.env, windowsHide: true, timeout: options?.timeoutMs }, (error, stdout, stderr) => { + execFile('git', args, { cwd, env: { ...process.env, ...options?.env }, windowsHide: true, timeout: options?.timeoutMs }, (error, stdout, stderr) => { const failed = error as NodeJS.ErrnoException & { status?: number } | null; const code = typeof failed?.status === 'number' ? failed.status : failed ? 1 : 0; resolve({ code, stdout: String(stdout), stderr: String(stderr) }); From 14c62c743621c86be0fce242f9ff67d6cae97577 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:14:59 +0200 Subject: [PATCH 47/77] fix(workflow): require an acting step to qualify delivery writes Inherited writable modes alone do not make approval and actionless steps writers. Count prompts and explicit writable workspaces when checking the delivery subtree. Red: approval-only and end-only delivery tests failed with workflow start accepting a subtree that writes nothing. --- src/__tests__/workflow-engine.test.ts | 18 ++++++++++++++++++ src/workflow/workspace.ts | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/__tests__/workflow-engine.test.ts b/src/__tests__/workflow-engine.test.ts index f7bdb96b1..92794474e 100644 --- a/src/__tests__/workflow-engine.test.ts +++ b/src/__tests__/workflow-engine.test.ts @@ -4491,6 +4491,24 @@ ${workspace} deliver: pr expect(await startRun(config, 'ship', { packages: ['web'] }, { base: { ref: 'main', commit } })).toMatchObject(expected); }); + it.each(['approve: Continue?', 'end: done'])('refuses delivery when an inherited writable workflow contains only approval or end steps (%s)', async (action) => { + const { started } = await start({ ship: workflow('ship', ` - id: change + workspace: new + deliver: pr + workflow: + ends: { done: succeeded } + steps: + - id: approve + ${action} + - id: check + needs: [] + command: echo check + end: done +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toMatchObject({ error: { code: 'bad_delivery', message: expect.stringContaining('no step of the workflow it runs writes a branch') } }); + }); + it('refuses a run that delivers from a detached HEAD, telling to start from a named branch', async () => { const { started } = await start({ ship: workflow('ship', ` - id: fix workspace: new diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 172c23f21..6729fe701 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -91,7 +91,9 @@ export function deliveryProblem( const childOf = (step: WorkflowStep) => (typeof step.workflow === 'string' ? workflows[step.workflow] : step.workflow); const writes = (steps: WorkflowStep[], ancestors: WorkflowStep[]): boolean => steps.some((step) => { const child = childOf(step); - return child ? writes(child.steps, [...ancestors, step]) : resolveMode(step, ancestors) !== 'read-only'; + if (child) return writes(child.steps, [...ancestors, step]); + return (step.prompt !== undefined || step.promptFile !== undefined || step.workspace === 'new' || step.workspace === 'continue') + && resolveMode(step, ancestors) !== 'read-only'; }); const visit = (steps: WorkflowStep[], ancestors: WorkflowStep[], prefix: string): WorkflowFailure | undefined => { for (const step of steps) { From c5ce06904600b927a1fba40ae3938966ba4f8d9a Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:20:04 +0200 Subject: [PATCH 48/77] refactor(workflow): keep delivery state and schemas in one ledger Move delivery selection and pure run updates into delivery-ledger.ts. The engine retains the drain and create gate and supplies reconciliation. Infer ledger types from the shared schemas and validate unresolved heads as full commit IDs. Red: npx tsc --noEmit failed with Cannot find module ./delivery-ledger.js after migrating the imports before moving the implementation. --- docs/dev/workflow-engine.md | 4 +- src/workflow/delivery-ledger.ts | 135 ++++++++++++++++++++++++++++++++ src/workflow/engine.ts | 112 ++------------------------ src/workflow/pr-delivery.ts | 16 +--- src/workflow/pr-url.ts | 10 --- src/workflow/snapshot.ts | 11 +-- src/workflow/types.ts | 3 +- 7 files changed, 146 insertions(+), 145 deletions(-) create mode 100644 src/workflow/delivery-ledger.ts diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index e476f76b3..728ca96e5 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,8 +13,8 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. -- **A pull request is reused by provider lookup, never by a stored receipt.** Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts new file mode 100644 index 000000000..c030a56d1 --- /dev/null +++ b/src/workflow/delivery-ledger.ts @@ -0,0 +1,135 @@ +/** The run's delivery receipts and unresolved requests, independent of locks and network calls. */ +import { z } from 'zod'; +import { PrUrlSchema, type PrUrl } from './pr-url.js'; +import { deliveryFailure, type DeliveryJob, type DeliveryOutcome, type findDelivered } from './pr-delivery.js'; +import { delivered, deliveryFailed, type DeliveringStep } from './step-state.js'; +import type { RunState } from './types.js'; +import { deliveryTarget } from './workspace.js'; + +/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ +export const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); + +/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ +export const DeliveryReceiptSchema = z.object({ + url: PrUrlSchema, + branch: z.string(), + base: z.string(), + step: z.string(), + pass: z.number().int(), +}); +export type DeliveryReceipt = z.infer; + +/** Which delivery an outcome belongs to while the step still delivers this pass of this commit. */ +const DeliveryKeySchema = z.object({ + step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: CommitIdSchema, +}); +export type DeliveryKey = z.infer; + +/** A create the host may have accepted, retained independently of the step's lifecycle. */ +export const UnresolvedCreateSchema = DeliveryKeySchema.extend({ + target: z.string().min(1), + lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), +}); +export type UnresolvedCreate = z.infer; + +/** The delivery of `step`, as the drain and the outcome it applies name it. */ +function deliveryKey(step: DeliveringStep): DeliveryKey { + return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; +} + +/** Whether `a` and `b` name one delivery: the same pass of the same step, settled at the same revision, of one commit. */ +function sameKey(a: DeliveryKey, b: DeliveryKey): boolean { + return a.step === b.step && a.pass === b.pass && a.settledAt === b.settledAt && a.branch === b.branch && a.head === b.head; +} + +/** The step `key` names, while it still delivers that pass of that commit. */ +export function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep | undefined { + return run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && sameKey(deliveryKey(item), key)); +} + +/** + * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel + * stopped is not delivered. Undefined when nothing delivers, or when the run settles, which the locked body of `next` + * finishes. + */ +export function nextDelivery(run: RunState, reconcile: (run: RunState) => object): DeliveryJob | undefined { + if (run.status !== 'running') return undefined; + const settledRun = reconcile(run); + if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; + for (const step of run.steps) { + if (step.status !== 'delivering') continue; + const key = deliveryKey(step); + const target = deliveryTarget(run, key.branch); + if (!run.checkout || !run.base || target === undefined) { + deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); + continue; + } + return { + key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, + recordedHead: run.copies?.[key.branch]?.commit, + ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), + }; + } + return undefined; +} + +/** The first request in `run.creating` that this drain has not looked up yet. */ +export function unresolvedCreate(run: RunState, tried: readonly Omit[]): { create: UnresolvedCreate; checkout: string } | undefined { + const { checkout } = run; + const create = run.creating?.find((listed) => !tried.some((done) => sameCreate(done, listed))); + return create && checkout ? { create, checkout } : undefined; +} + +export function sameCreate(a: Omit, b: Omit): boolean { + return sameKey(a, b) && a.target === b.target; +} + +/** Drops `create` from the requests the run has no receipt for. */ +function forgetCreate(run: RunState, create: Omit): void { + const left = run.creating?.filter((listed) => !sameCreate(listed, create)); + if (left?.length) run.creating = left; + else delete run.creating; +} + +/** Lists `url` as what `key` delivered into `target`, once; true when the run had no receipt for it yet. */ +function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrUrl): boolean { + if (run.deliveries?.some((receipt) => receipt.url === url)) return false; + (run.deliveries ??= []).push({ url, branch: key.branch, base: target, step: key.step, pass: key.pass }); + return true; +} + +/** + * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step + * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the + * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The + * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. + * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. + */ +export function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, reconcile: (run: RunState) => unknown): { requeueEnd: boolean } { + const receipted = outcome.ok && recordReceipt(run, job.key, job.target, outcome.pr); + if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); + if (run.status === 'running') reconcile(run); + const step = deliveringStep(run, job.key); + if (step && outcome.ok) delivered(step); + else if (step && !outcome.ok) deliveryFailed(step, deliveryFailure(job, outcome.fault)); + return { requeueEnd: receipted && run.status !== 'running' }; +} + +/** + * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, + * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. + */ +export function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited>): { requeueEnd: boolean } { + const listed = run.creating?.find((item) => sameCreate(item, create)); + if (!listed) return { requeueEnd: false }; + const lookups = listed.lookups ?? { total: 0, none: 0, failed: 0 }; + listed.lookups = { + total: lookups.total + 1, + none: lookups.none + (found.kind === 'none' ? 1 : 0), + failed: lookups.failed + (found.kind === 'failed' ? 1 : 0), + }; + const receipted = found.kind === 'found' && recordReceipt(run, create, create.target, found.pr); + if (found.kind === 'found' || found.kind === 'unsafe' || listed.lookups.none >= 2 || listed.lookups.failed >= 3) forgetCreate(run, create); + return { requeueEnd: receipted && run.status !== 'running' }; +} + diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index b11e33751..e7d87c639 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -18,17 +18,18 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - ReadSchema, answerHitl, cancel, delivered, deliveryFailed, fail, inProgress, lease, release, settled, skip, start, startDelivery, stopped, - succeed, waitForAnswer, type DeliveringStep, type ItemResult, type LeasedStep, type StepRead, type StepState, + ReadSchema, answerHitl, cancel, fail, inProgress, lease, release, settled, skip, start, startDelivery, stopped, + succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; import { - deliverPr, deliveryFailure, findDelivered, reportedPrMismatch, type DeliveryJob, type DeliveryKey, type DeliveryOutcome, type UnresolvedCreate, + deliverPr, findDelivered, reportedPrMismatch, type DeliveryJob, } from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; +import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate, type UnresolvedCreate } from './delivery-ledger.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; @@ -881,107 +882,6 @@ export interface NextOptions { claim?: boolean; } -/** The delivery of `step`, as the drain and the outcome it applies name it. */ -function deliveryKey(step: DeliveringStep): DeliveryKey { - return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; -} - -/** Whether `a` and `b` name one delivery: the same pass of the same step, settled at the same revision, of one commit. */ -function sameKey(a: DeliveryKey, b: DeliveryKey): boolean { - return a.step === b.step && a.pass === b.pass && a.settledAt === b.settledAt && a.branch === b.branch && a.head === b.head; -} - -/** The step `key` names, while it still delivers that pass of that commit. */ -function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep | undefined { - return run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && sameKey(deliveryKey(item), key)); -} - -/** - * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel - * stopped is not delivered. Undefined when nothing delivers, or when the run settles, which the locked body of `next` - * finishes. - */ -function nextDelivery(run: RunState): DeliveryJob | undefined { - if (run.status !== 'running') return undefined; - const settledRun = reconcileRun(run); - if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; - for (const step of run.steps) { - if (step.status !== 'delivering') continue; - const key = deliveryKey(step); - const target = deliveryTarget(run, key.branch); - if (!run.checkout || !run.base || target === undefined) { - deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); - continue; - } - return { - key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, - recordedHead: run.copies?.[key.branch]?.commit, - ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), - }; - } - return undefined; -} - -/** The first request in `run.creating` that this drain has not looked up yet. */ -function unresolvedCreate(run: RunState, tried: readonly Omit[]): { create: UnresolvedCreate; checkout: string } | undefined { - const { checkout } = run; - const create = run.creating?.find((listed) => !tried.some((done) => sameCreate(done, listed))); - return create && checkout ? { create, checkout } : undefined; -} - -function sameCreate(a: Omit, b: Omit): boolean { - return sameKey(a, b) && a.target === b.target; -} - -/** Drops `create` from the requests the run has no receipt for. */ -function forgetCreate(run: RunState, create: Omit): void { - const left = run.creating?.filter((listed) => !sameCreate(listed, create)); - if (left?.length) run.creating = left; - else delete run.creating; -} - -/** Lists `url` as what `key` delivered into `target`, once; true when the run had no receipt for it yet. */ -function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrUrl): boolean { - if (run.deliveries?.some((receipt) => receipt.url === url)) return false; - (run.deliveries ??= []).push({ url, branch: key.branch, base: target, step: key.step, pass: key.pass }); - return true; -} - -/** - * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step - * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the - * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The - * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. - * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. - */ -function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome): { requeueEnd: boolean } { - const receipted = outcome.ok && recordReceipt(run, job.key, job.target, outcome.pr); - if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); - if (run.status === 'running') reconcileRun(run); - const step = deliveringStep(run, job.key); - if (step && outcome.ok) delivered(step); - else if (step && !outcome.ok) deliveryFailed(step, deliveryFailure(job, outcome.fault)); - return { requeueEnd: receipted && run.status !== 'running' }; -} - -/** - * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, - * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. - */ -function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited>): { requeueEnd: boolean } { - const listed = run.creating?.find((item) => sameCreate(item, create)); - if (!listed) return { requeueEnd: false }; - const lookups = listed.lookups ?? { total: 0, none: 0, failed: 0 }; - listed.lookups = { - total: lookups.total + 1, - none: lookups.none + (found.kind === 'none' ? 1 : 0), - failed: lookups.failed + (found.kind === 'failed' ? 1 : 0), - }; - const receipted = found.kind === 'found' && recordReceipt(run, create, create.target, found.pr); - if (found.kind === 'found' || found.kind === 'unsafe' || listed.lookups.none >= 2 || listed.lookups.failed >= 3) forgetCreate(run, create); - return { requeueEnd: receipted && run.status !== 'running' }; -} - /** * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still * delivers `job`. If it does, the run lists its request in the same write, so one opened by a process that then stops @@ -1019,7 +919,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); - const job = nextDelivery(run); + const job = nextDelivery(run, reconcileRun); const value = job ? { job } : unresolvedCreate(run, tried); return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); @@ -1031,7 +931,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise markCreating(config, runId, job)); // As when the pick is busy: the step still delivers, and a later call delivers it. if (outcome === 'busy') return requeued; - apply = (run) => completeDelivery(run, job, outcome); + apply = (run) => completeDelivery(run, job, outcome, reconcileRun); } else { const { create, checkout } = picked.value; tried.push(create); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index b583d0536..5b780700e 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -7,24 +7,10 @@ import os from 'node:os'; import { providerForRemote, type OriginProvider } from '../providers/registry.js'; import type { PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-ledger.js'; import type { DeliveryFailure } from './step-state.js'; import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; -/** Which delivery an outcome belongs to: it applies only while the step still delivers this pass of this commit. */ -export interface DeliveryKey { - step: string; - pass: number; - settledAt: number; - branch: string; - head: string; -} - -/** - * A pull request TeamAI asked a host to open, into `target`, and holds no receipt for yet. The run lists it until a - * receipt or a lookup settles it, whatever becomes of its step, since the host may hold it. - */ -export type UnresolvedCreate = DeliveryKey & { target: string; lookups: { total: number; none: number; failed: number } }; - /** Everything a delivery needs, read from the run under its lock. */ export interface DeliveryJob { key: DeliveryKey; diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts index 67e967875..53e47dd22 100644 --- a/src/workflow/pr-url.ts +++ b/src/workflow/pr-url.ts @@ -38,13 +38,3 @@ export function sanitizePrUrl(value: unknown): { url: PrUrl; reason?: never } | const parsed = PrUrlSchema.safeParse(value); return parsed.success ? { url: parsed.data } : { reason: parsed.error.issues[0]?.message ?? 'is not a valid URL' }; } - -/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ -export const DeliveryReceiptSchema = z.object({ - url: PrUrlSchema, - branch: z.string(), - base: z.string(), - step: z.string(), - pass: z.number().int(), -}); -export type DeliveryReceipt = z.infer; diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index ad186d119..227deea6d 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -5,12 +5,11 @@ import { z } from 'zod'; import { canonicalJson } from '../utils/canonical-json.js'; import { redact } from '../utils/redact.js'; import { isRecord } from './expression.js'; -import { DeliveryReceiptSchema } from './pr-url.js'; +import { CommitIdSchema, DeliveryReceiptSchema, UnresolvedCreateSchema } from './delivery-ledger.js'; import { ArchivedPassSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; import type { RunState, StepLease, WorkflowDefinition, WorkflowFailure } from './types.js'; -import type { UnresolvedCreate } from './pr-delivery.js'; /** * The run state one install hands to another, at `workflows/runs//state.json` on `teamai-reports`. It carries what @@ -21,9 +20,6 @@ import type { UnresolvedCreate } from './pr-delivery.js'; const OwnerSchema = z.object({ id: z.string().min(1), user: z.string() }); export type HandoffOwner = z.infer; -/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ -const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); - const ReportedCommitsSchema = z.object({ changes: z.object({ base: CommitIdSchema, from: CommitIdSchema, to: CommitIdSchema, commits: z.array(CommitIdSchema) }).optional(), read: z.object({ base: CommitIdSchema, commit: CommitIdSchema }).optional(), @@ -38,11 +34,6 @@ function withCommitIds(schema: T): z.ZodEffects { }); } -const UnresolvedCreateSchema: z.ZodType = z.object({ - step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: z.string().min(1), target: z.string().min(1), - lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), -}); - const BodySchema = z.object({ schemaVersion: z.literal('v1alpha1'), name: z.string(), diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 047d443db..42eafb42a 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,8 +1,7 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; -import type { UnresolvedCreate } from './pr-delivery.js'; -import type { DeliveryReceipt } from './pr-url.js'; +import type { DeliveryReceipt, UnresolvedCreate } from './delivery-ledger.js'; import type { ArchivedPass, SessionContinuation, StepState } from './step-state.js'; import type { LeaseSession } from './workspace.js'; From 3ad890f308ea7e608b4eb49668750be5cade2586 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:21:15 +0200 Subject: [PATCH 49/77] style(workflow): remove trailing blank line from delivery ledger Keep the extracted module consistent with the surrounding files. Red: git diff 90363fbc3..HEAD --check failed with new blank line at EOF in delivery-ledger.ts. --- src/workflow/delivery-ledger.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index c030a56d1..c21c65a47 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -132,4 +132,3 @@ export function resolveCreate(run: RunState, create: UnresolvedCreate, found: Aw if (found.kind === 'found' || found.kind === 'unsafe' || listed.lookups.none >= 2 || listed.lookups.failed >= 3) forgetCreate(run, create); return { requeueEnd: receipted && run.status !== 'running' }; } - From bc349066c5967c5faaa5a447f837380a4eae252d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:25:41 +0200 Subject: [PATCH 50/77] fix(workflow): check the selected delivery tip against the run base Choose the reported or later recorded origin tip before reading its diff. A recorded correction that removes an earlier protected change can then deliver its clean final diff. Red: selected recorded tip test failed with delivery rejected after later work removed the earlier protected change. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 33 +++++++++++++++++++++++++ src/workflow/pr-delivery.ts | 20 ++++++--------- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 728ca96e5..36def3e10 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` fetches and diffs on the run's checkout, rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`), and requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It checks that later tip against the run base too, all before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index bc13c30bf..1c4ff89d7 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -915,6 +915,39 @@ describe('retrying and cancelling a delivery', () => { } }); + it('checks only the selected recorded tip when later work removed an earlier protected change', async () => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl, { '.github/workflows/temporary.yml': 'temporary' }) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const clone = fs.mkdtempSync(path.join(root, 'correction-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + git(clone, ['rm', '.github/workflows/temporary.yml']); + const to = commitAndPush(clone, branch, { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).deliveries).toHaveLength(1); + }); + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 5b780700e..5fae07874 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -68,22 +68,16 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { - const changed = await changedFiles(job); - if ('fault' in changed) return failed(changed.fault); - const files = protectedPaths(changed.files); - if (files.length > 0) return failed({ kind: 'protected_path', files }); - // A pull request holds what origin's branch holds, so it goes out only while that is the commit just checked. A push - // after this read still reaches a request opened below; only the driver that holds the branch can make that push. + if (!OBJECT_ID.test(job.key.head) || !OBJECT_ID.test(job.base)) return failed({ kind: 'unreadable' }); + // A later push can still change an open request after this read; only the driver holding the branch can make it. const tip = await originTip(job.checkout, job.key.branch); if (tip.kind === 'unreadable') return failed({ kind: 'branch_unreadable' }); if (tip.commit === undefined) return failed({ kind: 'branch_missing' }); - if (tip.commit !== job.key.head) { - if (tip.commit !== job.recordedHead) return failed({ kind: 'branch_moved' }); - const latest = await changedFiles({ ...job, key: { ...job.key, head: tip.commit } }); - if ('fault' in latest) return failed(latest.fault); - const protectedFiles = protectedPaths(latest.files); - if (protectedFiles.length > 0) return failed({ kind: 'protected_path', files: protectedFiles }); - } + if (tip.commit !== job.key.head && tip.commit !== job.recordedHead) return failed({ kind: 'branch_moved' }); + const changed = await changedFiles({ ...job, key: { ...job.key, head: tip.commit } }); + if ('fault' in changed) return failed(changed.fault); + const files = protectedPaths(changed.files); + if (files.length > 0) return failed({ kind: 'protected_path', files }); if (job.reported) return { ok: true, pr: job.reported }; const origin = await originOf(job.checkout); if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); From 743f4d58a4b8afb7ba38b887d94e292e98c6ecb0 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:52:00 +0200 Subject: [PATCH 51/77] fix(workflow): name the recorded head and wait for an ahead branch The branch_moved and branch_missing remedies named the head the delivering step reported. When a later report recorded a newer commit on the same branch, following them reset origin to the older commit and dropped the later step's settled work. Both remedies now name the commit the run recorded last. When origin's branch is past that commit and still contains it, a report of the newer commit may be on its way, so the step stays delivering. The drain does not pick it again, and the third drain in a row that finds the same tip fails the step with the branch_moved message. The count lives on the delivering step and the snapshot drops it. Red: "names the commit the run recorded last when the shared branch on origin is missing at a retry" failed with "push to it" instead of the recorded commit, and "keeps a step delivering while origin's branch is ahead" failed with done instead of wait. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 65 ++++++++++++++++++++++++- src/workflow/delivery-ledger.ts | 28 +++++++++-- src/workflow/engine.ts | 10 ++-- src/workflow/pr-delivery.ts | 24 +++++++-- src/workflow/snapshot.ts | 1 + src/workflow/step-state.ts | 12 +++-- 7 files changed, 123 insertions(+), 19 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 36def3e10..399bee1dd 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 1c4ff89d7..2b1ff8cc3 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -142,6 +142,21 @@ function pull(number: number, head: string, base = 'main', repo = 'fixture/app') const FIX_THEN_REVIEW = ` - id: fix\n workspace: new\n command: node fix.js\n deliver: pr\n - id: review\n needs: [fix]\n command: echo review\n`; +/** Two independent steps that continue one shared branch, the first delivering it. */ +const IMPL_AND_DOCS = ` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`; + describe('workflow PR delivery contract', () => { it('parses delivery on a writable step and rejects unsupported delivery', () => { const yaml = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: write\n command: node write.js\n workspace: new\n deliver: pr\n'; @@ -410,10 +425,11 @@ describe('delivering a step', () => { pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); expect(await succeed(config, runId, fix, { to: checked, ...changes })).not.toHaveProperty('data.dropped'); + for (const _drain of [1, 2]) expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['fix'] } } }); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const run = await state(config, runId); expect(run.steps).toMatchObject([ - { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin moved after the step reported commit ${checked}`) } }, + { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin moved away from commit ${checked}`) } }, { id: 'review', status: 'skipped' }, ]); expect(run.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); @@ -948,6 +964,53 @@ describe('retrying and cancelling a delivery', () => { expect((await state(config, runId)).deliveries).toHaveLength(1); }); + it.each(['moved', 'missing'] as const)('names the commit the run recorded last when the shared branch on origin is %s at a retry', async (change) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const docs = await take(config, runId); + const branch = docs.workspace?.branch ?? ''; + const recorded = pushOnto(root, branch, { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to: recorded }); + if (change === 'moved') pushOnto(root, branch, { 'later.txt': 'later' }); + else git(root, ['--git-dir=remote.git', 'update-ref', '-d', `refs/heads/${branch}`]); + gh.configure({}); + await retryStep(config, runId, 'work.impl'); + + const drains = change === 'moved' ? 3 : 1; + for (let drain = 1; drain < drains; drain += 1) { + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + } + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + const remedy = change === 'moved' ? `Put branch ${branch} on origin back at ${recorded}, then run` : `push ${recorded} to it, then run`; + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(remedy) } }); + expect(failure).not.toMatchObject({ failure: { message: expect.stringContaining(own) } }); + expect(creates(gh)).toEqual([]); + }); + + it('keeps a step delivering while origin\'s branch is ahead of what the run recorded, and delivers it once a report records that commit', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + expect(creates(gh)).toEqual([]); + await succeed(config, runId, docs, { to: ahead }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index c21c65a47..938292e40 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -49,16 +49,17 @@ export function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep /** * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel - * stopped is not delivered. Undefined when nothing delivers, or when the run settles, which the locked body of `next` - * finishes. + * stopped is not delivered. One in `waiting`, whose branch this drain found ahead of the recorded commit, waits for a + * later drain. Undefined when nothing delivers, or when the run settles, which the locked body of `next` finishes. */ -export function nextDelivery(run: RunState, reconcile: (run: RunState) => object): DeliveryJob | undefined { +export function nextDelivery(run: RunState, reconcile: (run: RunState) => object, waiting: readonly DeliveryKey[]): DeliveryJob | undefined { if (run.status !== 'running') return undefined; const settledRun = reconcile(run); if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; for (const step of run.steps) { if (step.status !== 'delivering') continue; const key = deliveryKey(step); + if (waiting.some((ahead) => sameKey(ahead, key))) continue; const target = deliveryTarget(run, key.branch); if (!run.checkout || !run.base || target === undefined) { deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); @@ -102,7 +103,8 @@ function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrU * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The - * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. + * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it, and not + * while origin's branch is ahead of the recorded commit and a report of it may still come. * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. */ export function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, reconcile: (run: RunState) => unknown): { requeueEnd: boolean } { @@ -111,10 +113,26 @@ export function completeDelivery(run: RunState, job: DeliveryJob, outcome: Deliv if (run.status === 'running') reconcile(run); const step = deliveringStep(run, job.key); if (step && outcome.ok) delivered(step); - else if (step && !outcome.ok) deliveryFailed(step, deliveryFailure(job, outcome.fault)); + else if (step && !outcome.ok && !(outcome.fault.kind === 'branch_ahead' && awaitsReport(step, outcome.fault.tip))) { + deliveryFailed(step, deliveryFailure(job, outcome.fault)); + } return { requeueEnd: receipted && run.status !== 'running' }; } +/** How many drains in a row may find origin's branch ahead of the recorded commit, at one commit, before delivery fails. */ +const AHEAD_DRAINS = 3; + +/** + * Whether `step` still waits for a report of `tip`, the commit origin's branch is ahead at: one may be on its way, from a + * step of the same branch that pushed before it reported. The drain that finds it there a third time fails the step. + */ +function awaitsReport(step: DeliveringStep, tip: string): boolean { + const drains = step.unrecordedTip?.commit === tip ? step.unrecordedTip.drains + 1 : 1; + if (drains >= AHEAD_DRAINS) return false; + step.unrecordedTip = { commit: tip, drains }; + return true; +} + /** * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index e7d87c639..1e3b65db8 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -29,7 +29,7 @@ import { deliverPr, findDelivered, reportedPrMismatch, type DeliveryJob, } from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; -import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate, type UnresolvedCreate } from './delivery-ledger.js'; +import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate, type DeliveryKey, type UnresolvedCreate } from './delivery-ledger.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; @@ -915,11 +915,12 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise { let requeued = false; const tried: Array> = []; + const waiting: DeliveryKey[] = []; for (;;) { const picked = await updateRun(config, runId, async (run) => { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); - const job = nextDelivery(run, reconcileRun); + const job = nextDelivery(run, reconcileRun, waiting); const value = job ? { job } : unresolvedCreate(run, tried); return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); @@ -931,6 +932,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise markCreating(config, runId, job)); // As when the pick is busy: the step still delivers, and a later call delivers it. if (outcome === 'busy') return requeued; + if (!outcome.ok && outcome.fault.kind === 'branch_ahead') waiting.push(job.key); apply = (run) => completeDelivery(run, job, outcome, reconcileRun); } else { const { create, checkout } = picked.value; @@ -984,8 +986,8 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv return answer(doneData(run)); }; const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => { - // Steps still delivering here are being delivered by another process of this machine, or waited for a busy run - // lock; a driver polls until they settle. + // Steps still delivering here are being delivered by another process of this machine, waited for a busy run + // lock, or wait for a report of the commit their branch is ahead at on origin; a driver polls until they settle. const delivering = run.steps.filter((step) => step.status === 'delivering').map((step) => step.id); const reason = open ? `${open.id} is leased` : delivering.length > 0 ? `${delivering[0]} is delivering` : 'canTakeLease is false'; return answer({ wait: { reason, ...(delivering.length > 0 ? { delivering } : {}) } }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 5fae07874..448f0ee11 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -34,6 +34,8 @@ export type DeliveryFault = | { kind: 'unreadable' } | { kind: 'protected_path'; files: string[] } | { kind: 'branch_moved' } + /** Origin's branch holds the recorded commit and more, at `tip`: a report of `tip` may still be on its way. */ + | { kind: 'branch_ahead'; tip: string } | { kind: 'branch_missing' } | { kind: 'branch_unreadable' } | { kind: 'no_provider' } @@ -73,7 +75,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { + if (!OBJECT_ID.test(commit) || !OBJECT_ID.test(ancestor)) return { contains: false }; + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [ancestor, commit] }); + if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; + return { contains: (await git(checkout, ['merge-base', '--is-ancestor', ancestor, commit])).code === 0 }; +} + const LISTED_FILES = 20; /** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ -export function deliveryFailure({ runId, key }: Pick, fault: DeliveryFault): DeliveryFailure { +export function deliveryFailure({ runId, key, recordedHead }: Pick, fault: DeliveryFault): DeliveryFailure { const { step, branch } = key; + const head = recordedHead ?? key.head; const retry = `teamai workflow retry ${runId} ${step}`; switch (fault.kind) { case 'no_checkout': @@ -213,12 +228,13 @@ export function deliveryFailure({ runId, key }: Pick; const PASS_RULES = { diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 37fd27f03..76bfc04e6 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -162,7 +162,11 @@ const PendingSchema = StepBaseSchema.extend({ status: z.literal('pending') }); const WaitingSchema = StepBaseSchema.extend({ status: z.literal('waiting') }); const RunningSchema = StepBaseSchema.extend({ status: z.literal('running') }); /** `delivering`: the agent succeeded and its work is on its branch; dependents wait until that branch has its pull request. */ -const DeliveringSchema = StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema }); +const DeliveringSchema = StepBaseSchema.extend({ + status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema, + /** The commit origin's branch was found ahead at, past the recorded one, and how many drains in a row found it there. */ + unrecordedTip: z.object({ commit: z.string(), drains: z.number().int().positive() }).optional(), +}); const SucceededSchema = StepBaseSchema.extend({ status: z.literal('succeeded'), outcome: z.literal('success').default('success'), ...settledFields, loopFinal: z.boolean().optional(), }); @@ -298,7 +302,7 @@ export function startDelivery(state: StepState, settlement: Settlement): Fail */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { if (state.status === 'delivering') { - const { id: _id, status: _status, ...settlement } = state; + const { id: _id, status: _status, unrecordedTip: _tip, ...settlement } = state; return become(state, { ...settlement, status: 'cancelled', failure }); } const attempt = state.status === 'leased' ? state.attempt : undefined; From da1def374921e5814ac38443bffc6d3aea2ae13f Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:54:48 +0200 Subject: [PATCH 52/77] fix(workflow): blame a later commit for its own protected change When origin's branch was at a later recorded commit, the protected-path check diffed only that tip. A protected change another step's later commit made failed the delivering step with protected_path, whose retry runs the step's agent again and fails the same way. When the tip is a later recorded commit and it changes protected paths, the step's own head is diffed too. A clean own head fails with delivery_failed, naming the files and the later commit, so a retry delivers again without running the agent. A protected own head still fails with protected_path and lists the step's own files. Red: "retries delivery at a later recorded shared-branch tip and checks its protected paths (true)" failed with protected_path instead of delivery_failed, and "blames the delivering step for its own protected change" listed the later commit's .github/workflows/docs.yml. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 22 +++++++++++++++++++++- src/workflow/pr-delivery.ts | 25 ++++++++++++++++++++----- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 399bee1dd..3bbfae60b 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index cadfb9184..b5c510ad1 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -240,7 +240,7 @@ A pull request goes into the nearest branch its own branch grew from that alread Each pass of a looped `workspace: new` step has its own `/p` branch, so it opens its own pull request. The earlier ones stay open, by design. A step that loops on one `continue` branch has one pull request. -Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. +Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. When only a later commit that another step recorded on a shared branch made the change, the step fails with `delivery_failed` instead, and the retry does not run it again. Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 8d8380d70..f939db3f1 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -240,7 +240,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 循环中 `workspace: new` 步骤的每个 pass 都有自己的 `/p` 分支，因此各开一个 pull request。之前的 pull request 保持打开，这是有意为之。在同一条 `continue` 分支上循环的步骤只有一个 pull request。 -开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。 +开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。如果修改只来自另一个步骤之后在共享分支上记录的提交，步骤改为以 `delivery_failed` 失败，重试也不会重新运行它。 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 2b1ff8cc3..79205b877 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -921,9 +921,12 @@ describe('retrying and cancelling a delivery', () => { expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: protectedChange ? 'failed' : 'succeeded' } } }); const run = await state(config, runId); if (protectedChange) { - expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining(file) } }); + const message = `commit ${to}, which a later report recorded on the shared branch ${docs.workspace?.branch}, changed protected paths against the run base: ${file}. Step "work.impl" itself left them alone, so a retry does not run it again.`; + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(message) } }); expect(creates(gh)).toEqual([]); expect(run.deliveries).toBeUndefined(); + await retryStep(config, runId, 'work.impl'); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering', attempt: impl.attempt }); } else { expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); expect(creates(gh)).toEqual([[docs.workspace?.branch, 'main']]); @@ -931,6 +934,23 @@ describe('retrying and cancelling a delivery', () => { } }); + it('blames the delivering step for its own protected change when a later report recorded more work on the shared branch', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl, { '.github/workflows/impl.yml': 'impl' }) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + await succeed(config, runId, docs, { to: pushOnto(root, docs.workspace?.branch ?? '', { '.github/workflows/docs.yml': 'docs' }) }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('Step "work.impl" changed protected paths against the run base: .github/workflows/impl.yml.') } }); + expect(creates(gh)).toEqual([]); + }); + it('checks only the selected recorded tip when later work removed an earlier protected change', async () => { const { config, root, runId } = await fixture(` - id: work workspace: new diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 448f0ee11..61220b8e5 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -33,6 +33,8 @@ export type DeliveryFault = | { kind: 'no_checkout' } | { kind: 'unreadable' } | { kind: 'protected_path'; files: string[] } + /** Only the commits a later report recorded after the step's own head change `files`. */ + | { kind: 'later_protected_path'; files: string[] } | { kind: 'branch_moved' } /** Origin's branch holds the recorded commit and more, at `tip`: a report of `tip` may still be on its way. */ | { kind: 'branch_ahead'; tip: string } @@ -83,6 +85,12 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise 0 && tip.commit !== job.key.head) { + const own = await changedFiles(job); + if ('fault' in own) return failed(own.fault); + const owned = protectedPaths(own.files); + return failed(owned.length > 0 ? { kind: 'protected_path', files: owned } : { kind: 'later_protected_path', files }); + } if (files.length > 0) return failed({ kind: 'protected_path', files }); if (job.reported) return { ok: true, pr: job.reported }; const origin = await originOf(job.checkout); @@ -201,6 +209,11 @@ async function contains({ runId, checkout }: DeliveryJob, { commit, ancestor }: const LISTED_FILES = 20; +function listedFiles(files: string[]): string { + const more = files.length > LISTED_FILES ? `, and ${files.length - LISTED_FILES} more` : ''; + return `${files.slice(0, LISTED_FILES).join(', ')}${more}`; +} + /** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ export function deliveryFailure({ runId, key, recordedHead }: Pick, fault: DeliveryFault): DeliveryFailure { const { step, branch } = key; @@ -217,14 +230,16 @@ export function deliveryFailure({ runId, key, recordedHead }: Pick LISTED_FILES ? `, and ${fault.files.length - LISTED_FILES} more` : ''; + case 'protected_path': return { code: 'protected_path', - message: `Step "${step}" changed protected paths against the run base: ${listed}${more}. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. Have the step leave those paths alone, then run: ${retry}`, + message: `Step "${step}" changed protected paths against the run base: ${listedFiles(fault.files)}. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. Have the step leave those paths alone, then run: ${retry}`, + }; + case 'later_protected_path': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: commit ${head}, which a later report recorded on the shared branch ${branch}, changed protected paths against the run base: ${listedFiles(fault.files)}. Step "${step}" itself left them alone, so a retry does not run it again. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. Have a step on that branch take those changes out, then run: ${retry}`, }; - } case 'branch_missing': return { code: 'delivery_failed', From 0b476cc696a064e09eaa58a22de4b50892e1d952 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 18:58:37 +0200 Subject: [PATCH 53/77] fix(workflow): require a recorded tip to contain the step's head A later recorded tip was delivered as soon as it matched the copy's recorded commit. A driver that rewrote the shared branch after the step, for example by amending its commit and force-pushing, then had a pull request opened, and the step marked succeeded, without the step's work. A tip other than the reported head must now contain it. Otherwise delivery fails with delivery_failed, naming the recorded commit and the missing one, and opens or accepts nothing. A head that the run's checkout cannot fetch while it holds the tip is not in the tip's history. The brief asked for branch_moved. Its remedy names the recorded commit, which origin already holds in this case, so following it would change nothing; the failure has its own message instead. Red: "refuses a recorded later tip whose history no longer holds the delivering step's commit" failed with outcome succeeded instead of failed. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 28 +++++++++++++++++++++++++ src/workflow/pr-delivery.ts | 16 +++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 3bbfae60b..dbac4f571 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 79205b877..4faff3b15 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -951,6 +951,34 @@ describe('retrying and cancelling a delivery', () => { expect(creates(gh)).toEqual([]); }); + it('refuses a recorded later tip whose history no longer holds the delivering step\'s commit', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const clone = fs.mkdtempSync(path.join(root, 'rewrite-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + git(clone, ['reset', '-q', '--hard', `${own}~1`]); + fs.writeFileSync(path.join(clone, 'docs.txt'), 'docs'); + git(clone, ['add', '-A']); + git(clone, ['-c', 'user.name=Driver', '-c', 'user.email=driver@example.com', 'commit', '-qm', 'rewrite']); + git(clone, ['push', '-q', '--force', 'origin', branch]); + const rewritten = git(clone, ['rev-parse', 'HEAD']); + await succeed(config, runId, docs, { to: rewritten }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${rewritten}, the last one a report recorded on branch ${branch}, does not contain commit ${own} the step reported`) } }); + expect(creates(gh)).toEqual([]); + expect((await state(config, runId)).deliveries).toBeUndefined(); + }); + it('checks only the selected recorded tip when later work removed an earlier protected change', async () => { const { config, root, runId } = await fixture(` - id: work workspace: new diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 61220b8e5..95d96cf27 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -38,6 +38,8 @@ export type DeliveryFault = | { kind: 'branch_moved' } /** Origin's branch holds the recorded commit and more, at `tip`: a report of `tip` may still be on its way. */ | { kind: 'branch_ahead'; tip: string } + /** The recorded commit origin's branch is at lacks the step's head: a later step rewrote the branch's history. */ + | { kind: 'branch_rewritten' } | { kind: 'branch_missing' } | { kind: 'branch_unreadable' } | { kind: 'no_provider' } @@ -82,6 +84,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { if (!OBJECT_ID.test(commit) || !OBJECT_ID.test(ancestor)) return { contains: false }; const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [ancestor, commit] }); - if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; + if (fetched.missing.includes(commit)) return { fault: { kind: 'unreadable' } }; + // A commit the checkout holds brings its history, so an ancestor still missing is not in it. + if (fetched.missing.length > 0) return { contains: false }; return { contains: (await git(checkout, ['merge-base', '--is-ancestor', ancestor, commit])).code === 0 }; } @@ -251,6 +260,11 @@ export function deliveryFailure({ runId, key, recordedHead }: Pick Date: Sat, 24 Oct 2026 19:00:39 +0200 Subject: [PATCH 54/77] fix(workflow): read the delivery tip with the shared non-interactive git env The tip ls-remote set only GIT_TERMINAL_PROMPT=0 and credential.interactive=false. Git still runs an askpass helper when one is configured, such as the one an editor terminal exports, so an expired HTTPS credential could open a dialog on every drain. The ls-remote now runs with NON_INTERACTIVE_GIT_ENV from git-env.ts, which also closes the askpass chain and Git Credential Manager, and keeps -c credential.interactive=false. The guides no longer claim which prompts the check closes. Red: "names a missing origin branch and reads it without credential prompts" failed because ls-remote was called with env { GIT_TERMINAL_PROMPT: '0' } only. --- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 3 ++- src/utils/git-env.ts | 7 ++++--- src/workflow/pr-delivery.ts | 3 ++- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index b5c510ad1..f43c21e54 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -244,7 +244,7 @@ Before it opens or accepts a pull request, TeamAI compares the delivered commit Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. -If origin has no branch to deliver, TeamAI opens no request. The failure names the missing branch and the commit to push to it, then gives the retry command. The origin check disables terminal and credential-manager prompts. +If origin has no branch to deliver, TeamAI opens no request. The failure names the missing branch and the commit to push to it, then gives the retry command. If a host may have opened the request but TeamAI cannot confirm it, the step fails with `delivery_failed` and records no URL. Run the retry command in the message. TeamAI looks for the request before opening one again. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index f939db3f1..5dbb7f015 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -244,7 +244,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 -如果 origin 没有要交付的分支，TeamAI 不会打开请求。失败消息会指出缺失的分支和要推送到该分支的提交，然后给出重试命令。检查 origin 时会禁用终端和凭据管理器的提示。 +如果 origin 没有要交付的分支，TeamAI 不会打开请求。失败消息会指出缺失的分支和要推送到该分支的提交，然后给出重试命令。 如果主机可能已打开请求，但 TeamAI 无法确认，步骤会以 `delivery_failed` 失败，且不记录 URL。运行消息中的重试命令。TeamAI 会先查找已有请求，再尝试打开请求。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 4faff3b15..7434ee860 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -447,12 +447,13 @@ describe('delivering a step', () => { git(root, ['--git-dir=remote.git', 'update-ref', '-d', `refs/heads/${branch}`]); await succeed(config, runId, fix, { to }); vi.stubEnv('GIT_TERMINAL_PROMPT', '1'); + vi.stubEnv('GIT_ASKPASS', '/opt/editor/askpass.sh'); const reads = vi.spyOn(worktree, 'git'); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const run = await state(config, runId); expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`origin has no branch ${branch}; push ${to} to it, then run: teamai workflow retry ${runId} fix`) } }); - expect(reads).toHaveBeenCalledWith(cwd, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', `refs/heads/${branch}`], { timeoutMs: 30000, env: { GIT_TERMINAL_PROMPT: '0' } }); + expect(reads).toHaveBeenCalledWith(cwd, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', `refs/heads/${branch}`], { timeoutMs: 30000, env: { GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo', GCM_INTERACTIVE: 'never' } }); expect(run.deliveries).toBeUndefined(); expect(gh.calls()).toEqual([]); }); diff --git a/src/utils/git-env.ts b/src/utils/git-env.ts index 6c10f7c3a..cf0f3719f 100644 --- a/src/utils/git-env.ts +++ b/src/utils/git-env.ts @@ -23,10 +23,11 @@ import { isInteractive } from './prompt.js'; /** - * Applied only where the caller left the variable unset, so an explicit value - * always wins. + * Applied process-wide only where the caller left the variable unset, so an + * explicit value always wins there. A git child that must never prompt, even + * in an interactive run, gets it as its own environment. */ -const NON_INTERACTIVE_GIT_ENV: Record = { +export const NON_INTERACTIVE_GIT_ENV: Readonly> = { GIT_TERMINAL_PROMPT: '0', // `echo` answers the askpass request with an empty line, so the credential is // rejected at once instead of a dialog waiting for a person who is not there. diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 95d96cf27..64ef8d9a6 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -5,6 +5,7 @@ */ import os from 'node:os'; import { providerForRemote, type OriginProvider } from '../providers/registry.js'; +import { NON_INTERACTIVE_GIT_ENV } from '../utils/git-env.js'; import type { PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import type { DeliveryKey, UnresolvedCreate } from './delivery-ledger.js'; @@ -190,7 +191,7 @@ async function findOpen( /** The commit origin's `branch` is at, read by the checkout; none when origin has no such branch. */ async function originTip(checkout: string, branch: string): Promise<{ kind: 'read'; commit?: string } | { kind: 'unreadable' }> { const ref = `refs/heads/${branch}`; - const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS, env: { GIT_TERMINAL_PROMPT: '0' } }); + const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS, env: NON_INTERACTIVE_GIT_ENV }); if (listed.code !== 0) return { kind: 'unreadable' }; const commit = listed.stdout.split('\n').map((line) => line.split('\t')).find(([, name]) => name?.trim() === ref)?.[0]; return commit ? { kind: 'read', commit } : { kind: 'read' }; From e5da6a0bba043baa558da4ab7d91e32c25c230dc Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 19:05:36 +0200 Subject: [PATCH 55/77] fix(workflow): count a retried create's lookups from zero A retried delivery keeps its key, so the gate found the earlier failed create's entry in run.creating and left its lookup counts alone. Counts from before the retry then applied to the new create, and one empty lookup after it could drop the request as its second. The gate now sets the entry's counts to zero each time it opens, since a new create was just sent. Red: "counts the lookups of a retried create from zero" failed because the entry still had { total: 1, none: 1 } after the new create. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 20 ++++++++++++++++++++ src/workflow/engine.ts | 6 +++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index dbac4f571..ae5dba3f9 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -14,7 +14,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. - **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. -- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, initialized by the gate and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. +- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 7434ee860..e1038a386 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1263,6 +1263,26 @@ describe('retrying and cancelling a delivery', () => { expect(lookup).toHaveBeenCalledTimes(4); }); + it('counts the lookups of a retried create from zero, so one empty lookup after it keeps the request listed', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const lookup = vi.fn(async () => undefined); + const create = providerDouble(lookup, async () => { + throw new Error('create timed out'); + }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + + await retryStep(config, runId, 'fix'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 0, none: 0, failed: 0 } })]); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + expect(create).toHaveBeenCalledTimes(2); + }); + it('records a pull request a cancelled delivery opened before its process stopped, by looking it up on a later call, and opens no other', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); config.usageReportEnabled = true; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 1e3b65db8..ea5cc2b9e 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -894,7 +894,11 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); if (!step) return { value: 'refused' }; const create = { ...job.key, target: job.target }; - if (!run.creating?.some((listed) => sameCreate(listed, create))) (run.creating ??= []).push({ ...create, lookups: { total: 0, none: 0, failed: 0 } }); + const listed = run.creating?.find((item) => sameCreate(item, create)); + // A request listed before is asked again now, so its lookups count from this create. + const lookups = { total: 0, none: 0, failed: 0 }; + if (listed) listed.lookups = lookups; + else (run.creating ??= []).push({ ...create, lookups }); return { value: 'open', write: { ...run, updatedAt: new Date().toISOString() } }; }); return marked.status === 'done' ? marked.value : 'busy'; From ad25dc83a77920d5e332c191e2f75985efae5967 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 19:09:46 +0200 Subject: [PATCH 56/77] refactor(workflow): keep delivery schemas in a leaf module The ledger extraction moved the receipt and unresolved-create schemas, and CommitIdSchema, into delivery-ledger.ts, which imports pr-delivery.ts as a value. snapshot.ts, and through it the run store, records and handoff, then loaded every provider adapter at module load. delivery-schema.ts now holds those schemas and their types, importing only zod and pr-url.ts. The ledger keeps the pure updates. A source scan in no-provider-merge.test.ts follows snapshot.ts's runtime imports and fails if they reach providers/ or pr-delivery.ts. Red: "the run snapshot loads neither delivery code nor a provider" failed because the walk reached workflow/pr-delivery.ts and 27 other modules, providers/registry.ts and every adapter among them. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/no-provider-merge.test.ts | 17 ++++++++++++++ src/workflow/delivery-ledger.ts | 30 ++----------------------- src/workflow/delivery-schema.ts | 29 ++++++++++++++++++++++++ src/workflow/engine.ts | 3 ++- src/workflow/pr-delivery.ts | 2 +- src/workflow/snapshot.ts | 2 +- src/workflow/types.ts | 2 +- 8 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 src/workflow/delivery-schema.ts diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index ae5dba3f9..235166e19 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -14,7 +14,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. - **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. -- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-ledger.ts` owns the schemas and pure updates for delivery receipts and unresolved creates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. +- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. diff --git a/src/__tests__/no-provider-merge.test.ts b/src/__tests__/no-provider-merge.test.ts index 384c3d9c7..fc49eb2d2 100644 --- a/src/__tests__/no-provider-merge.test.ts +++ b/src/__tests__/no-provider-merge.test.ts @@ -40,4 +40,21 @@ describe('workflow delivery never merges', () => { it('delivery code imports nothing from the engine, which applies its outcome', () => { expect(fs.readFileSync(path.join(SRC, 'workflow', 'pr-delivery.ts'), 'utf8')).not.toMatch(/from '\.\/engine\.js'/); }); + + it('the run snapshot loads neither delivery code nor a provider', () => { + const loaded = [...runtimeImports(path.join('workflow', 'snapshot.ts'))]; + expect(loaded.filter((file) => file.startsWith(`providers${path.sep}`) || file === path.join('workflow', 'pr-delivery.ts'))).toEqual([]); + expect(loaded).toContain(path.join('workflow', 'delivery-schema.ts')); + }); }); + +/** `file` and every module it loads, following each relative import or re-export that is not `import type`. */ +function runtimeImports(file: string, seen = new Set()): Set { + if (seen.has(file)) return seen; + seen.add(file); + const text = fs.readFileSync(path.join(SRC, file), 'utf8'); + for (const [, specifier] of text.matchAll(/^(?:import|export)(?!\s+type\b)[^;]*?\sfrom\s+'(\.[^']*)\.js'/gm)) { + runtimeImports(path.join(path.dirname(file), `${specifier}.ts`), seen); + } + return seen; +} diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index 938292e40..facc10d52 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -1,37 +1,11 @@ /** The run's delivery receipts and unresolved requests, independent of locks and network calls. */ -import { z } from 'zod'; -import { PrUrlSchema, type PrUrl } from './pr-url.js'; +import type { PrUrl } from './pr-url.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import { deliveryFailure, type DeliveryJob, type DeliveryOutcome, type findDelivered } from './pr-delivery.js'; import { delivered, deliveryFailed, type DeliveringStep } from './step-state.js'; import type { RunState } from './types.js'; import { deliveryTarget } from './workspace.js'; -/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ -export const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); - -/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ -export const DeliveryReceiptSchema = z.object({ - url: PrUrlSchema, - branch: z.string(), - base: z.string(), - step: z.string(), - pass: z.number().int(), -}); -export type DeliveryReceipt = z.infer; - -/** Which delivery an outcome belongs to while the step still delivers this pass of this commit. */ -const DeliveryKeySchema = z.object({ - step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: CommitIdSchema, -}); -export type DeliveryKey = z.infer; - -/** A create the host may have accepted, retained independently of the step's lifecycle. */ -export const UnresolvedCreateSchema = DeliveryKeySchema.extend({ - target: z.string().min(1), - lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), -}); -export type UnresolvedCreate = z.infer; - /** The delivery of `step`, as the drain and the outcome it applies name it. */ function deliveryKey(step: DeliveringStep): DeliveryKey { return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; diff --git a/src/workflow/delivery-schema.ts b/src/workflow/delivery-schema.ts new file mode 100644 index 000000000..0adc84a15 --- /dev/null +++ b/src/workflow/delivery-schema.ts @@ -0,0 +1,29 @@ +/** What a run file and a snapshot keep of delivery, as schemas only, so neither loads delivery code or a provider. */ +import { z } from 'zod'; +import { PrUrlSchema } from './pr-url.js'; + +/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ +export const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); + +/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ +export const DeliveryReceiptSchema = z.object({ + url: PrUrlSchema, + branch: z.string(), + base: z.string(), + step: z.string(), + pass: z.number().int(), +}); +export type DeliveryReceipt = z.infer; + +/** Which delivery an outcome belongs to while the step still delivers this pass of this commit. */ +const DeliveryKeySchema = z.object({ + step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: CommitIdSchema, +}); +export type DeliveryKey = z.infer; + +/** A create the host may have accepted, retained independently of the step's lifecycle. */ +export const UnresolvedCreateSchema = DeliveryKeySchema.extend({ + target: z.string().min(1), + lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), +}); +export type UnresolvedCreate = z.infer; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index ea5cc2b9e..e40b87787 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -29,7 +29,8 @@ import { deliverPr, findDelivered, reportedPrMismatch, type DeliveryJob, } from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; -import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate, type DeliveryKey, type UnresolvedCreate } from './delivery-ledger.js'; +import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate } from './delivery-ledger.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 64ef8d9a6..b21f87b09 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -8,7 +8,7 @@ import { providerForRemote, type OriginProvider } from '../providers/registry.js import { NON_INTERACTIVE_GIT_ENV } from '../utils/git-env.js'; import type { PullRequestRef } from '../providers/types.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; -import type { DeliveryKey, UnresolvedCreate } from './delivery-ledger.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import type { DeliveryFailure } from './step-state.js'; import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 684a642f2..d88955ff3 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { canonicalJson } from '../utils/canonical-json.js'; import { redact } from '../utils/redact.js'; import { isRecord } from './expression.js'; -import { CommitIdSchema, DeliveryReceiptSchema, UnresolvedCreateSchema } from './delivery-ledger.js'; +import { CommitIdSchema, DeliveryReceiptSchema, UnresolvedCreateSchema } from './delivery-schema.js'; import { ArchivedPassSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 42eafb42a..5500e07bb 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,7 +1,7 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; -import type { DeliveryReceipt, UnresolvedCreate } from './delivery-ledger.js'; +import type { DeliveryReceipt, UnresolvedCreate } from './delivery-schema.js'; import type { ArchivedPass, SessionContinuation, StepState } from './step-state.js'; import type { LeaseSession } from './workspace.js'; From 504b9546427985c3d01fec3f057a1c2583d63775 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 19:15:05 +0200 Subject: [PATCH 57/77] fix(workflow): treat an empty lookup after a create as unconfirmed When the lookup right after a create found no open request, delivery recorded the URL the create answered with, as long as it was on the origin's host. A TGit create prints only a number, so the provider builds a /-/merge_requests/N URL, and the list API may not show the new request yet. A later delivery of the branch then found the canonical URL and recorded the same merge request a second time. An empty lookup after a create that answered now fails with create_unconfirmed, records no receipt, and keeps the unresolved entry, as a failed lookup does. A later drain's lookup records the canonical URL. The create's own URL is no longer read. Two tests whose lookup double never listed the request it had just created now list it, since their subject is the run lock and a cancelled run, not host lag. Red: "records only the canonical receipt when the lookup right after a TGit-style create does not list it yet" failed because the create URL was recorded and review was leased instead of delivery failing. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 37 +++++++++++++++++++++++-- src/workflow/pr-delivery.ts | 19 ++++++------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 235166e19..fcdb2f24e 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. A failed post-create lookup fails delivery without recording the create URL and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index e1038a386..2b6c49a84 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1092,6 +1092,31 @@ describe('retrying and cancelling a delivery', () => { expect(lookup).toHaveBeenCalledTimes(3); }); + it('records only the canonical receipt when the lookup right after a TGit-style create does not list it yet', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const canonical = 'https://git.woa.com/fixture/app/merge_requests/7'; + let listed = false; + const lookup = vi.fn(async () => (listed ? canonical : undefined)); + const create = providerDouble(lookup, async () => 'https://git.woa.com/fixture/app/-/merge_requests/7', 'git.woa.com'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`may have opened a pull request from branch ${fix.workspace?.branch}, but TeamAI could not confirm it`) } }); + expect(failed.deliveries).toBeUndefined(); + expect(failed.creating).toHaveLength(1); + + listed = true; + await leaseNext(config, runId, remote, writable); + await retryStep(config, runId, 'fix'); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([canonical]); + expect(create).toHaveBeenCalledTimes(1); + }); + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); @@ -1155,10 +1180,14 @@ describe('retrying and cancelling a delivery', () => { } return result; }); + let opened: string | undefined; const create = providerDouble(async () => { if (!released && !holding) holding = await acquireLock(runLock); - return undefined; - }, async () => 'https://github.com/fixture/app/pull/1'); + return opened; + }, async () => { + opened = 'https://github.com/fixture/app/pull/1'; + return opened; + }); expect(await leaseNext(config, runId, remote, writable)).toEqual({ data: { wait: { reason: 'fix is delivering', delivering: ['fix'] }, cancelled: [] } }); expect(released).toBe(true); @@ -1192,8 +1221,10 @@ describe('retrying and cancelling a delivery', () => { const fix = await take(config, runId); await succeed(config, runId, fix, { to: pushWork(root, fix) }); const url = 'https://github.com/fixture/app/pull/8'; - providerDouble(async () => undefined, async () => { + let opened: string | undefined; + providerDouble(async () => opened, async () => { await cancelRun(config, runId); + opened = url; return url; }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index b21f87b09..81a23a252 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -112,28 +112,25 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise | undefined; + let answered = false; try { - created = sanitizePrUrl(await provider.createPullRequest({ + await provider.createPullRequest({ repo, source: job.key.branch, target: job.target, title: `${job.runName}: ${job.key.step}`, description: `Opened by TeamAI for review: workflow ${job.runName}, run ${job.runId}, step ${job.key.step}, pass ${job.key.pass}.\n\nBase commit: ${job.base}\n\nTeamAI never merges this pull request.`, // Run outside the member checkout, so the provider's CLI can neither push from it nor read its branch. cwd: os.tmpdir(), spawnTimeoutMs: DELIVERY_TIMEOUT_MS, - })); + }); + answered = true; } catch { - created = undefined; + // A create that failed or timed out may still have opened the request, so the lookup below decides. } - // A create that failed or timed out may still have opened the request. One that answered may name it by another URL - // than the lookup a later delivery of the branch finds it by, so the lookup's is kept: one request, one receipt. + // One that answered may name the request by another URL than the lookup a later delivery of the branch finds it by, + // or before the host lists it, so only the lookup's URL is kept: one request, one receipt. const opened = await findOpen(origin, ref); if (opened.kind === 'found') return { ok: true, pr: opened.pr }; if (opened.kind === 'unsafe') return failed({ kind: 'unsafe_url', provider: provider.name }); - if (opened.kind === 'failed') return failed({ kind: 'create_unconfirmed', provider: provider.name }); - if (!created) return failed({ kind: 'create_failed', provider: provider.name }); - return created.url && new URL(created.url).hostname === origin.hostname - ? { ok: true, pr: created.url } - : failed({ kind: 'unsafe_url', provider: provider.name }); + return failed({ kind: answered || opened.kind === 'failed' ? 'create_unconfirmed' : 'create_failed', provider: provider.name }); } /** From ff6a7f6f52d9dd08c17873c980bed319c25efb0b Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 19:22:04 +0200 Subject: [PATCH 58/77] docs(workflow): name deliverPr as the subject of the tip diff The bounded-wait sentences landed between deliverPr's tip rule and "It fetches and diffs that selected tip", so "It" read as the remedies. Red: none; a wording correction found in the whole-round doc review. --- docs/dev/workflow-engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index fcdb2f24e..5d3c0b70f 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. It fetches and diffs that selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. From e12d9b9f815c5a3c3206ab679a0880e760fa9c60 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:12:04 +0800 Subject: [PATCH 59/77] feat(workflow): add durable pull request delivery - Add deliver: pr for writable leaves, composite branches, fan-out items and loop passes. Persist post-agent delivery separately, serialize provider I/O outside the run lock, retain cancellation receipts and retry failed delivery without rerunning the agent or losing the pushed branch. - Record copy origins and resolve stacked targets through after dependencies, skipped/read-only steps and inherited workflow needs. Reuse open requests through existing GitHub, GitLab, TGit, GitCode and CNB authentication; retain the generic Git fallback and never invoke a provider merge API. - Inspect the exact delivered commit against the run base before PR creation, reject protected CI/agent/MCP/rules/hook paths, and preserve execution bounds across delivery. Reconcile limits reached by a report before provider I/O. - Validate and sanitize driver review URLs without exposing rejected values, report unknown changes fields by name, and publish ordered, deduplicated prs from validated changes only. Keep usage-report opt-out and done envelopes. - Document delivery and the unattended CI path in both languages, update help and dashboard state, and verify 1,311 related unit tests, 40 CLI E2E cases, focused provider/recovery/security tests, typecheck, lint and combined requirements/session/delivery integration against local fixture remotes. --- docs/guide/workflow.md | 51 ++- docs/guide/zh-CN/workflow.md | 51 ++- skill-data/core/references/commands.md | 2 +- .../e2e/workflow-conformance.test.ts | 4 +- .../e2e/workflow-pr-delivery-1090.test.ts | 102 ++++++ src/__tests__/workflow-conformance.test.ts | 4 +- src/__tests__/workflow-delivery.test.ts | 293 ++++++++++++++++++ src/__tests__/workflow-docs-examples.test.ts | 2 + src/__tests__/workflow-pr-providers.test.ts | 97 ++++++ src/__tests__/workflow-records.test.ts | 4 +- src/dashboard/client.ts | 2 +- src/dashboard/locales.ts | 1 + src/dashboard/workflow-runs.ts | 4 +- src/index.ts | 2 +- src/providers/cnb/cnb-cli.ts | 8 +- src/providers/cnb/index.ts | 4 + src/providers/cnb/workflow-pr.ts | 41 +++ src/providers/gitcode/gitcode-api.ts | 2 + src/providers/gitcode/index.ts | 4 + src/providers/gitcode/workflow-pr.ts | 37 +++ src/providers/github/gh-cli.ts | 1 + src/providers/github/index.ts | 9 + src/providers/github/workflow-pr.ts | 48 +++ src/providers/gitlab/gitlab-api.ts | 2 + src/providers/gitlab/index.ts | 10 + src/providers/gitlab/workflow-pr.ts | 42 +++ src/providers/tgit/index.ts | 4 + src/providers/tgit/workflow-pr.ts | 32 ++ src/providers/types.ts | 6 + src/workflow/conformance.ts | 30 +- src/workflow/delivery.ts | 192 ++++++++++++ src/workflow/engine.ts | 117 ++++++- src/workflow/fan-out.ts | 2 +- src/workflow/local-driver.ts | 4 + src/workflow/parse.ts | 11 +- src/workflow/pr-url.ts | 16 + src/workflow/records.ts | 12 + src/workflow/run-controls.ts | 10 +- src/workflow/step-state.ts | 22 +- src/workflow/types.ts | 7 +- src/workflow/workspace.ts | 18 +- 41 files changed, 1267 insertions(+), 43 deletions(-) create mode 100644 src/__tests__/e2e/workflow-pr-delivery-1090.test.ts create mode 100644 src/__tests__/workflow-delivery.test.ts create mode 100644 src/__tests__/workflow-pr-providers.test.ts create mode 100644 src/providers/cnb/workflow-pr.ts create mode 100644 src/providers/gitcode/workflow-pr.ts create mode 100644 src/providers/github/workflow-pr.ts create mode 100644 src/providers/gitlab/workflow-pr.ts create mode 100644 src/providers/tgit/workflow-pr.ts create mode 100644 src/workflow/delivery.ts create mode 100644 src/workflow/pr-url.ts diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c94439dca..c05b7d796 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -173,7 +173,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. `used` (`{"skills":["review-notes"]}`) names the team skills the pass loaded, for its [scorer labels](scorers.md#any-driver). A value that is not a skill name is dropped, and a `used` of another shape returns `invalid_report`. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. +`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. `used` (`{"skills":["review-notes"]}`) names the team skills the pass loaded, for its [scorer labels](scorers.md#any-driver). A value that is not a skill name is dropped, and a `used` of another shape returns `invalid_report`. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` and `changes.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. The run is stored in this scope's TeamAI data directory. The command exits when it finishes. A local run that ends `failed` or `cancelled` exits with status 1. No workflow process keeps running. @@ -235,8 +235,57 @@ When a published run ends, teamai publishes how it ended, without `--sessions`. A step with `session: continue` or `session.from` starts a new agent session on the other machine, unless you publish with `--sessions`. When a run goes back and forth, every machine that continued a session publishes with `--sessions` before the next claim, so the next step gets every turn. [Publish and claim](workflow-advanced.md#publish-and-claim) covers what the published state carries, every error and its next step, and publishing from CI. +## Deliver a PR + +Set `deliver: pr` on a writable step, or on a `workflow` step whose children share its `workspace: new` branch. TeamAI pushes the branch and opens a review request through the product repository's Git provider. It never merges it. A `for_each` delivers one PR per item. A looped `new` step gets a new `/p` branch and PR each pass; earlier PRs stay open. + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +The PR targets the nearest delivered branch along the branch's recorded origin; `for_each.after` comes before other `needs`, and the first child inherits the enclosing workflow's dependencies. Skipped or read-only dependencies are passed over. If no earlier branch delivered, the target is the run's original branch. Start from a named branch: a detached checkout has no PR target. An open PR for the same head and target is reused. + +Agent completion is followed by a durable `delivering` state. Dependents wait for it. Git and provider operations happen outside the run lock. A PR creation failure reports `delivery_failed` and names the preserved branch. Fix provider authentication, run `teamai workflow retry `, then `teamai workflow resume `: this retries delivery on that branch without rerunning the agent. Push failures remain `push_failed` and retain the worktree. A completed composite may already have removed its worktree; its committed, pushed branch remains. Generic Git hosts support transport only: create a PR yourself and report its URL, or use a provider that supports PR creation. No interactive login starts during delivery. + +Before opening a PR, TeamAI checks the delivered commit against the run's base commit. Changes to CI, hooks, agent instructions, rules, MCP and equivalent tool configuration are rejected, with the paths named. The shared agent configuration paths are the same ones restored before workspace execution. This check includes changes inherited from earlier branches and deletions/renames. + +A remote driver may include `changes.pr_url` alongside `branch`, `base`, `from`, `to` and `commits`. Recognized provider metadata is checked against the expected repository and branches. Any HTTPS host/path is allowed for private review services. URLs containing credentials, invalid/non-HTTPS URLs, and URLs longer than 512 characters are dropped; query strings and fragments are stripped. A dropped URL does not reject the report: `dropped` lists `{ field: "changes.pr_url", reason: "…" }` without the value. Unknown nested fields are listed in `ignored` as `changes.`. Drivers must still obey a lease's workspace and commit contract. + +### Start from CI + +Store the workflow above in the team repo and configure a local prompt tool there. A normal GitHub Actions job is the v1 trigger path. The token must read the team repo, push branches to the product repo, and create PRs there. Configure `TEAM_REPO` and `WORKFLOW_TOKEN` in the repository settings. + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` answers optional approve requests and optional choose defaults only. A required gate fails with `hitl_required` in an unattended job. To start from a failing branch, check that branch out by name; the default `pull_request` checkout is a detached merge commit. TeamAI does not install triggers or wait for CI/merges. + ## Run records +The end record includes `prs`, the validated `changes.pr_url` links in first-appearance order, once per URL across items and loop passes. URLs from step outputs are never published. `prs` follows the same usage-report opt-out as the rest of the record and is public when the team repo is public. Older records without `prs` remain readable. + With [usage reporting](reporting.md) on, every run publishes a privacy-preserving record to the team's `teamai-reports` branch: once when it starts and once when it ends. With it off, nothing is published and the envelopes carry no `record`. Records live in `workflows/runs/.json`, keyed by run id, so a retried publication updates the same record instead of adding a copy. A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 43062c079..9feea5fdc 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -173,7 +173,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。`used`（`{"skills":["review-notes"]}`）写明该轮次加载过的团队技能，供它的[评分标签](scorers.md#任意-driver)使用。不是技能名的值会被丢弃，其他形状的 `used` 返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 +`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。`used`（`{"skills":["review-notes"]}`）写明该轮次加载过的团队技能，供它的[评分标签](scorers.md#任意-driver)使用。不是技能名的值会被丢弃，其他形状的 `used` 返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>` 和 `changes.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 运行状态保存在当前 scope 的 TeamAI 数据目录里。命令结束后进程就退出。本地运行以 `failed` 或 `cancelled` 结束时，退出码是 1。不会留下常驻的工作流进程。 @@ -235,8 +235,57 @@ teamai workflow resume # 在另一台机器上，位于同一个仓 带 `session: continue` 或 `session.from` 的步骤在另一台机器上会开一个新的 agent 会话，除非发布时加上 `--sessions`。运行在机器之间来回交接时，每台继续过会话的机器都要在下一次认领前用 `--sessions` 发布，这样下一个步骤能拿到所有对话轮次。[发布和认领](workflow-advanced.md#发布和认领)说明发布的状态包含什么、每个错误及下一步怎么做，以及如何从 CI 发布。 +## 交付 PR + +在可写步骤上设置 `deliver: pr`，或在子步骤共用其 `workspace: new` 分支的 `workflow` 步骤上设置它。TeamAI 推送分支，通过产品仓库的 Git provider 创建供人审查的 PR，从不合并。`for_each` 每项交付一个 PR。循环中的 `new` 步骤每个 pass 使用新的 `/p` 分支和 PR，之前的 PR 保持打开。 + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +PR 的目标沿分支记录的来源，追溯到最近已经交付的分支。`for_each.after` 优先于其他 `needs`，第一个子步骤继承外层工作流的依赖；跳过或只读的依赖会继续向前追溯。之前没有交付分支时，目标为运行开始时的分支。因此请从有名称的分支开始；detached checkout 没有 PR 目标。同一源分支和目标分支已有打开的 PR 时会复用。 + +agent 完成后，步骤进入持久的 `delivering` 状态，后续依赖等待交付结束。Git 和 provider 操作不占用运行锁。创建 PR 失败时以 `delivery_failed` 结束，消息写明保留的分支。修好认证后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，只重试这条分支的交付，不重跑 agent。推送失败仍为 `push_failed`，并保留 worktree。组合工作流结束时可能已移除 worktree，但提交和远程分支仍在。通用 Git 主机只支持传输：请自行创建 PR 并上报 URL，或使用支持创建 PR 的 provider。交付不会启动交互登录。 + +创建 PR 前，TeamAI 将交付提交与运行的基准提交比较。CI、hooks、agent 指令、rules、MCP 和同类工具配置的修改会被拒绝，并列出路径。共享 agent 配置路径与工作区执行前恢复的路径相同。检查也覆盖从前序分支继承的修改，以及删除和重命名。 + +远程驱动可在 `changes` 的 `branch`、`base`、`from`、`to`、`commits` 之外上报 `pr_url`。provider 能提供元数据时，会校验仓库和源、目标分支。私有审查服务可使用任意 HTTPS 主机和路径。带凭据、无效、非 HTTPS 或超过 512 字符的 URL 被丢弃；query 和 fragment 会移除。丢弃 URL 不会拒绝 report，`dropped` 以 `{ field: "changes.pr_url", reason: "…" }` 列出原因，不包含值。未知嵌套字段以 `changes.<字段>` 列入 `ignored`。驱动仍须遵守租约的工作区与提交契约。 + +### 从 CI 启动 + +将上面的工作流放进团队仓库，并在其中配置本地提示词工具。普通 GitHub Actions 作业是 v1 的触发方式。token 需要能读取团队仓库、向产品仓库推送分支和创建 PR。在仓库设置里配置 `TEAM_REPO` 和 `WORKFLOW_TOKEN`。 + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` 只回答可选 approve 和有默认值的可选 choose。必需的 gate 在无人值守作业中以 `hitl_required` 失败。从失败分支启动时，按分支名检出；默认的 `pull_request` checkout 是 detached merge commit。TeamAI 不安装触发器，也不等待 CI 或合并事件。 + ## 运行记录 +结束记录包含 `prs`：通过校验的 `changes.pr_url` 按首次出现顺序排列，跨 item 和循环 pass 去重。步骤 outputs 中的 URL 不会发布。`prs` 遵循相同的 usage-report 退出设置；团队仓库公开时它也公开。没有 `prs` 的旧记录仍可读取。 + 开启[使用统计上报](reporting.md)时，每次运行都会向团队的 `teamai-reports` 分支发布一条保护隐私的记录：开始时一次，结束时一次。关闭后不发布任何记录，返回中也不带 `record`。记录存放在 `workflows/runs/.json`，以 run id 为键，重试的发布会更新同一条记录而不是产生副本。 记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数，以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 7432b947c..67e2c5cc0 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -220,7 +220,7 @@ Generated: do not edit by hand. Regenerate with ## workflow -- `teamai workflow` — Run a validated workflow from workflows/.yaml +- `teamai workflow` — Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them. - `teamai workflow start ` — Create a run. With --driver local, execute command and prompt steps in this process. - `--driver ` — Run the workflow in this process. Only local is available. - `--tools ` — Override local prompt tools for this run, as comma-separated name=tool pairs diff --git a/src/__tests__/e2e/workflow-conformance.test.ts b/src/__tests__/e2e/workflow-conformance.test.ts index fc67c2e77..62e106f6d 100644 --- a/src/__tests__/e2e/workflow-conformance.test.ts +++ b/src/__tests__/e2e/workflow-conformance.test.ts @@ -75,7 +75,7 @@ describe('workflow protocol conformance through the CLI', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -84,6 +84,7 @@ describe('workflow protocol conformance through the CLI', () => { ...(operation.running ? { running: operation.running.map(attemptOf) } : {}), ...(operation.canTakeLease !== undefined ? { canTakeLease: operation.canTakeLease } : {}), ...(operation.claim !== undefined ? { claim: operation.claim } : {}), + ...(operation.workspace ? { driver: { ...driver, can: { workspace: ['read-only', 'new', 'continue'] } }, base: { ref: 'main', commit: 'a'.repeat(40) } } : {}), }); expect(value).toMatchObject(operation.expect); if (operation.saveAs === 'busy') { @@ -114,6 +115,7 @@ describe('workflow protocol conformance through the CLI', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, + ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), }); expect(value).toMatchObject(operation.expect); } diff --git a/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts new file mode 100644 index 000000000..59535baf7 --- /dev/null +++ b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts @@ -0,0 +1,102 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { trackDetachedProcesses } from '../helpers/detached-processes.js'; +import type { RunState } from '../../workflow/types.js'; + +const cli = fileURLToPath(new URL('../../../dist/index.js', import.meta.url)); + +describe('workflow PR delivery CLI (#1090)', () => { + it('runs the documented CI path, creates/reuses a provider PR, retains generic-git work, and refuses required unattended approvals', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-pr-cli-'))); + const home = path.join(root, 'home'); const product = path.join(root, 'product'); const team = path.join(root, 'team'); const bin = path.join(root, 'bin'); + for (const dir of [home, product, team, bin]) fs.mkdirSync(dir); + const detached = trackDetachedProcesses(root); + const env: NodeJS.ProcessEnv = { ...process.env, HOME: home, USERPROFILE: home, PATH: `${bin}${path.delimiter}${process.env.PATH}`, FORCE_COLOR: '0', + GIT_CONFIG_GLOBAL: path.join(home, '.gitconfig'), GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', + GIT_AUTHOR_NAME: 'Fixture', GIT_AUTHOR_EMAIL: 'fixture@example.invalid', GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + NODE_OPTIONS: [process.env.NODE_OPTIONS, detached.nodeOptions].filter(Boolean).join(' '), PR_FIXTURE: path.join(root, 'pr.json') }; + delete env.CLAUDE_CONFIG_DIR; delete env.CODEX_HOME; delete env.GITHUB_TOKEN; delete env.GH_TOKEN; + const run = (command: string, args: string[], cwd = product) => spawnSync(command, args, { cwd, env, encoding: 'utf8', timeout: 30_000 }); + const git = (args: string[], cwd = product) => { const result = run('git', args, cwd); expect(result.status, result.stderr).toBe(0); return result.stdout.trim(); }; + const call = (args: string[]) => run(process.execPath, [cli, ...args]); + const envelope = (stdout: string) => JSON.parse(stdout.trim().split('\n').filter((line) => line.startsWith('{')).at(-1)!); + const productUrl = 'https://github.com/fixture/app.git'; const teamUrl = 'https://team.example.invalid/fixture/team.git'; + const bare = path.join(root, 'product.git'); + fs.writeFileSync(path.join(bin, 'gh'), `#!${process.execPath} +const fs = require('fs'); const args = process.argv.slice(2); const file = process.env.PR_FIXTURE; +if (args[0] === '--version') { console.log('gh version 2.80.0'); process.exit(0); } +if (args[0] === 'api') { + const saved = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null; + console.log(JSON.stringify(args[1].includes('?') ? saved ? [saved] : [] : saved)); process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'create') { + const val = key => args[args.indexOf(key) + 1]; + if (val('-R') !== 'fixture/app' || val('-B') !== 'main') process.exit(7); + const pr = { state: 'open', html_url: 'https://github.com/fixture/app/pull/41', head: { ref: val('-H'), repo: { full_name: 'fixture/app' } }, base: { ref: val('-B'), repo: { full_name: 'fixture/app' } } }; + fs.writeFileSync(file, JSON.stringify(pr)); fs.appendFileSync(file + '.creates', 'create\\n'); console.log(pr.html_url); process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'list') { console.log('[]'); process.exit(0); } +console.error('Unexpected provider operation: ' + args[0]); process.exit(9); +`, { mode: 0o755 }); + try { + git(['init', '-q', '-b', 'main']); fs.writeFileSync(path.join(product, 'README.md'), '# Product\n'); git(['add', '.']); git(['commit', '-qm', 'base']); + git(['init', '-q', '--bare', '-b', 'main', bare]); git(['remote', 'add', 'origin', productUrl]); + git(['config', '--global', `url.${bare}.insteadOf`, productUrl]); git(['push', '-qu', 'origin', 'main']); + git(['init', '-q', '-b', 'main'], team); fs.mkdirSync(path.join(team, 'workflows')); + fs.writeFileSync(path.join(team, 'teamai.yaml'), `team: fixture\nrepo: ${teamUrl}\nprovider: git\nusageReport: false\n`); + const yaml = `schemaVersion: v1alpha1 +name: ship-issue +description: Deliver an issue change for review. +inputs: { issue: number } +steps: + - id: approve + approve: { message: Start the fixture?, required: false } + - id: work + needs: [approve] + workspace: new + deliver: pr + command: node -e "require('fs').writeFileSync('issue.txt', process.env.ISSUE)" + env: + ISSUE: "{{ inputs.issue }}" +`; + fs.writeFileSync(path.join(team, 'workflows', 'ship-issue.yaml'), yaml); + fs.writeFileSync(path.join(team, 'workflows', 'required.yaml'), yaml.replace('name: ship-issue', 'name: required').replace('required: false', 'required: true')); + git(['add', '.'], team); git(['commit', '-qm', 'team workflows'], team); + git(['config', '--global', `url.${team}.insteadOf`, teamUrl]); + const initialized = call(['init', teamUrl, '--scope', 'workspace', '--agent', 'claude', '--force']); expect(initialized.status, `${initialized.stdout}\n${initialized.stderr}`).toBe(0); + const pulled = call(['pull']); expect(pulled.status, pulled.stderr).toBe(0); await detached.waitForExit(); + const started = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":42}']); + expect(started.status, `${started.stdout}\n${started.stderr}`).toBe(0); + const done = envelope(started.stdout).data.done; expect(done.outcome).toBe('succeeded'); + const runFile = fs.readdirSync(root, { recursive: true, encoding: 'utf8' }).find((file) => file.endsWith(path.join('workflows', 'runs', `${done.run}.json`))); + expect(runFile).toBeDefined(); + const dataHome = path.dirname(path.dirname(path.dirname(path.join(root, runFile!)))); + const read = (id: string) => JSON.parse(fs.readFileSync(path.join(dataHome, 'workflows', 'runs', `${id}.json`), 'utf8')) as RunState; + const state = read(done.run); const work = state.steps.find((step) => step.id === 'work'); + expect(work).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/41' } }); + const branch = `teamai/${done.run}/work/p1`; + expect(git(['--git-dir', bare, 'show', `${branch}:issue.txt`])).toBe('42'); expect(fs.existsSync(path.join(product, 'issue.txt'))).toBe(false); + // Re-enter durable post-agent delivery as after a crash: the open PR is reused, with no new agent work or create. + if (work?.status !== 'succeeded') throw new Error('missing successful work'); + const persisted = state as unknown as { status: string; steps: Array> }; + persisted.status = 'running'; const target = persisted.steps.find((step) => step.id === 'work')!; target.status = 'delivering'; delete target.outcome; + delete (target.changes as Record).pr_url; delete (target.changes as Record).prRecordedAt; + fs.writeFileSync(path.join(dataHome, 'workflows', 'runs', `${done.run}.json`), JSON.stringify(persisted)); + const resumed = call(['workflow', 'resume', done.run]); expect(resumed.status, resumed.stderr).toBe(0); + expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); + git(['remote', 'set-url', 'origin', bare]); + const generic = call(['workflow', 'start', 'ship-issue', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":43}']); + expect(generic.status).toBe(1); const genericDone = envelope(generic.stdout).data.done; + const failed = read(genericDone.run).steps.find((step) => step.id === 'work'); const failedBranch = `teamai/${genericDone.run}/work/p1`; + expect(failed).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(failedBranch) } }); + expect(git(['--git-dir', bare, 'show', `${failedBranch}:issue.txt`])).toBe('43'); + const required = call(['workflow', 'start', 'required', '--driver', 'local', '--auto-approve', '--inputs', '{"issue":44}']); + expect(required.status).toBe(1); expect(read(envelope(required.stdout).data.done.run).steps[0]).toMatchObject({ failure: { code: 'hitl_required' } }); + expect(fs.readFileSync(`${env.PR_FIXTURE}.creates`, 'utf8')).toBe('create\n'); + } finally { await detached.waitForExit(); fs.rmSync(root, { recursive: true, force: true }); } + }); +}); diff --git a/src/__tests__/workflow-conformance.test.ts b/src/__tests__/workflow-conformance.test.ts index aa01cd5ff..e140cd3a1 100644 --- a/src/__tests__/workflow-conformance.test.ts +++ b/src/__tests__/workflow-conformance.test.ts @@ -64,7 +64,7 @@ describe('workflow protocol conformance in process', () => { const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { if (operation.kind === 'next') { @@ -73,6 +73,7 @@ describe('workflow protocol conformance in process', () => { ...(operation.running ? { running: operation.running.map(attemptOf) } : {}), ...(operation.canTakeLease !== undefined ? { canTakeLease: operation.canTakeLease } : {}), ...(operation.claim !== undefined ? { claim: operation.claim } : {}), + ...(operation.workspace ? { driver: { ...driver, can: { workspace: ['read-only', 'new', 'continue'] } }, base: { ref: 'main', commit: 'a'.repeat(40) } } : {}), }); expect(envelope).toMatchObject(operation.expect); if (operation.saveAs === 'busy') { @@ -101,6 +102,7 @@ describe('workflow protocol conformance in process', () => { revision: operation.revision ?? lease?.revision, driver, ...operation.body, + ...(operation.leasedChanges ? { changes: { branch: lease!.workspace!.branch, base: lease!.workspace!.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...(operation.body.changes as Record) } } : {}), }); expect(envelope).toMatchObject(operation.expect); } diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts new file mode 100644 index 000000000..7ab3666af --- /dev/null +++ b/src/__tests__/workflow-delivery.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fc from 'fast-check'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { parseWorkflow } from '../workflow/parse.js'; +import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; +import { sanitizePrUrl } from '../workflow/pr-url.js'; +import { deliveryTarget, protectedDeliveryPaths, validateReportedPr } from '../workflow/delivery.js'; +import { syncWorkflowDefinitions } from '../workflow/cache.js'; +import { readRun, writeRun } from '../workflow/store.js'; +import { runLocalDriver } from '../workflow/local-driver.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import { buildEndRecord } from '../workflow/records.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import { settled, stopped } from '../workflow/step-state.js'; +import type { LocalConfig } from '../types.js'; +import type { RunState } from '../workflow/types.js'; + +const dirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function git(cwd: string, args: string[]): string { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim(); +} + +async function fixture(steps: string, inputs?: unknown): Promise<{ config: LocalConfig; cwd: string; runId: string; root: string }> { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-delivery-')); + dirs.push(root); + const cwd = path.join(root, 'product'); + const knowledge = path.join(root, 'team'); + fs.mkdirSync(cwd); + fs.mkdirSync(path.join(knowledge, 'workflows'), { recursive: true }); + git(root, ['init', '-q', '--bare', 'remote.git']); + git(cwd, ['init', '-q', '-b', 'main']); + git(cwd, ['config', 'user.name', 'Test']); + git(cwd, ['config', 'user.email', 'test@example.com']); + fs.writeFileSync(path.join(cwd, '.gitignore'), '.teamai/\n'); + fs.writeFileSync(path.join(cwd, 'base.txt'), 'base\n'); + git(cwd, ['add', '-A']); git(cwd, ['commit', '-qm', 'base']); + git(cwd, ['remote', 'add', 'origin', 'https://github.com/fixture/app.git']); + git(cwd, ['config', `url.${path.join(root, 'remote.git')}.insteadOf`, 'https://github.com/fixture/app.git']); + git(cwd, ['push', '-qu', 'origin', 'main']); + const config: LocalConfig = { repo: { localPath: knowledge, remote: 'local', kind: 'self' }, scope: 'project', + username: 'tester', usageReportEnabled: false, additionalRoles: [], projectRoot: cwd, dataHome: path.join(root, 'data'), workflows: { tools: {}, max_parallel: 1 } }; + fs.writeFileSync(path.join(knowledge, 'workflows', 'ship.yaml'), `schemaVersion: v1alpha1\nname: ship\ndescription: Deliver work\n${inputs ? 'inputs:\n packages: [{ name: string, id: string, after: [string] }]\n' : ''}steps:\n${steps}`); + await syncWorkflowDefinitions(config); + const started = await startRun(config, 'ship', inputs ?? {}, { checkout: cwd, base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) } }); + if ('error' in started) throw new Error(started.error.message); + return { config, cwd, root, runId: String(started.data.run) }; +} + +async function state(config: LocalConfig, runId: string): Promise { + const run = await readRun(config, runId); + if (typeof run === 'string') throw new Error(run); + return run; +} + +const command = `node -e "require('fs').writeFileSync('change.txt', 'change')"`; +const driver = { kind: 'local', machine: 'fixture', workspace: 'fixture' }; + +function provider() { + const lookup = vi.spyOn(GitHubProvider.prototype, 'findOpenPullRequest').mockResolvedValue(undefined); + const validate = vi.spyOn(GitHubProvider.prototype, 'validatePullRequest').mockResolvedValue(true); + let number = 0; + const create = vi.spyOn(GitHubProvider.prototype, 'createPullRequest').mockImplementation(async () => `https://github.com/fixture/app/pull/${++number}`); + return { lookup, validate, create }; +} + +describe('workflow PR delivery contract', () => { + it('parses delivery on a writable step and rejects unsupported delivery', () => { + const yaml = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: write\n command: node write.js\n workspace: new\n deliver: pr\n'; + const parsed = parseWorkflow('ship.yaml', yaml); + expect(parsed.ok && parsed.definition.steps[0]).toMatchObject({ deliver: 'pr' }); + expect(parseWorkflow('ship.yaml', yaml.replace('deliver: pr', 'deliver: merge')).ok).toBe(false); + }); + + it('keeps the URL for validation and names unread nested fields without values', () => { + const parsed = parseReport({ attempt: 'a1', revision: 1, outcome: 'success', + driver: { kind: 'remote', machine: 'host', workspace: 'work' }, + changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', private: 'secret' } }); + expect(parsed).toMatchObject({ report: { changes: { pr_url: 'https://review.example/1' } }, ignored: ['changes.private'] }); + expect(JSON.stringify(parsed)).not.toContain('secret'); + }); + + it.each([ + ['https://github.com/acme/app/pull/101', 'https://github.com/acme/app/pull/101'], + ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview'], + ['https://review.acme.internal/changes/4821?session=abc#c3', 'https://review.acme.internal/changes/4821'], + ['https://user:token@github.com/acme/app/pull/1', undefined], + ['http://github.com/acme/app/pull/1', undefined], ['not a url', undefined], + [`https://example.com/${'a'.repeat(512)}`, undefined], + ])('sanitizes %s', (value, expected) => expect(sanitizePrUrl(value).url).toBe(expected)); + + it('never preserves credentials, queries or fragments and is idempotent and bounded', () => { + fc.assert(fc.property(fc.webUrl({ validSchemes: ['https', 'http'] }), (value) => { + const result = sanitizePrUrl(value); + if (!result.url) return; + const url = new URL(result.url); + expect(url.protocol).toBe('https:'); + expect([url.username, url.password, url.search, url.hash]).toEqual(['', '', '', '']); + expect(result.url.length).toBeLessThanOrEqual(512); + expect(sanitizePrUrl(result.url)).toEqual(result); + }), { numRuns: 300 }); + }); +}); + +describe('durable PR delivery', () => { + it('pushes a branch and opens a PR against the recorded base without merging', async () => { + const api = provider(); + const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/1' } }); + expect(api.create).toHaveBeenCalledWith(expect.objectContaining({ repo: 'fixture/app', source: `teamai/${runId}/write/p1`, target: 'main' })); + expect(git(root, ['--git-dir=remote.git', 'show', `refs/heads/teamai/${runId}/write/p1:change.txt`])).toBe('change'); + expect(git(cwd, ['show', 'main:base.txt'])).toBe('base'); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('keeps pushed work after missing credentials and retries delivery without rerunning the agent', async () => { + const api = provider(); + api.lookup.mockRejectedValue(new Error('Set GITHUB_TOKEN before resuming.')); + const { config, cwd, root, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); + let run = await state(config, runId); + const branch = `teamai/${runId}/write/p1`; + expect(run.steps[0]).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(branch) } }); + expect(git(root, ['--git-dir=remote.git', 'show', `${branch}:change.txt`])).toBe('change'); + const attempts = run.settledCount; + api.lookup.mockResolvedValue('https://github.com/fixture/app/pull/88'); + expect(await retryStep(config, runId, 'write')).toMatchObject({ data: { reopened: true } }); + run = await state(config, runId); + expect(run.steps[0]?.status).toBe('delivering'); + expect(settled(run.steps[0])).toBe(false); + expect(stopped(run.steps[0]!)).toBe(false); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + run = await state(config, runId); + expect(run.settledCount).toBe(attempts); + expect(api.create).not.toHaveBeenCalled(); + expect(run.steps[0]).toMatchObject({ changes: { branch, pr_url: 'https://github.com/fixture/app/pull/88' } }); + }); + + it('checks protected changes against the run base before any provider request', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/evil.yml','bad')"\n workspace: new\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/evil.yml') } }); + expect(api.create).not.toHaveBeenCalled(); expect(api.lookup).not.toHaveBeenCalled(); + expect(protectedDeliveryPaths(['src/a.ts', '.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit'])).toEqual(['.mcp.json', '.cursor/rules/x', 'nested/AGENTS.md', '.husky/pre-commit']); + }); + + it('delivers a continued composite branch once and follows inherited needs', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: change\n needs: [first]\n workspace: new\n deliver: pr\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').writeFileSync('second.txt','second')"\n - id: review\n needs: [write]\n command: cat second.txt\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create).toHaveBeenCalledTimes(2); + expect(api.create.mock.calls[1]?.[0]).toMatchObject({ source: `teamai/${runId}/change/p1`, target: `teamai/${runId}/first/p1` }); + expect((await state(config, runId)).steps.find((step) => step.id === 'change')).toMatchObject({ status: 'succeeded', changes: { pr_url: 'https://github.com/fixture/app/pull/2' } }); + }); + + it('delivers each after-item against its actual origin and publishes PRs in item order', async () => { + const api = provider(); + const { config, cwd, root, runId } = await fixture(` - id: packages\n for_each:\n items: inputs.packages\n workspace: new\n deliver: pr\n command: node -e "require('fs').writeFileSync(process.env.PACKAGE+'.txt',process.env.PACKAGE)"\n env:\n PACKAGE: "{{ item.name }}"\n`, { packages: [{ name: 'one', id: 'one', after: [] }, { name: 'two', id: 'two', after: ['one'] }, { name: 'three', id: 'three', after: ['two'] }] }); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls.map(([opts]) => opts.target)).toEqual(['main', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`]); + expect(git(root, ['--git-dir=remote.git', 'diff', '--name-only', `teamai/${runId}/packages-0/p1`, `teamai/${runId}/packages-1/p1`])).toBe('two.txt'); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); + }); + + it('accepts sanitized remote links, drops unsafe/mismatched ones without exposing their values', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); + const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'], base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) } }); + const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; + if (!lease) throw new Error(JSON.stringify(next)); + const changes = { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: 'https://user:token@github.com/fixture/app/pull/1' }; + const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes }); + expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); + expect(JSON.stringify([accepted, await state(config, runId)])).not.toContain('token'); + expect(api.validate).not.toHaveBeenCalled(); + }); + + it.each([ + { url: 'https://review.example/changes/1?session=secret#note', valid: undefined, expected: 'https://review.example/changes/1', reason: undefined }, + { url: 'https://github.com/fixture/app/pull/1', valid: false, expected: undefined, reason: 'repository or branches do not match the lease' }, + ])('records only a validated driver link ($url)', async ({ url, valid, expected, reason }) => { + const api = provider(); api.validate.mockResolvedValue(valid); + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n`); + const next = await leaseNext(config, runId, driver, { workspaceModes: ['new'] }); + const lease = 'data' in next && next.data.lease as { attempt: string; revision: number; workspace: { branch: string; base: { commit: string } } }; + if (!lease) throw new Error('missing lease'); + const accepted = await reportAttempt(config, runId, { attempt: lease.attempt, revision: lease.revision, driver, outcome: 'success', changes: { + branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a', to: 'b', commits: [], pr_url: url, + } }); + expect(accepted).toMatchObject({ data: { accepted: true, ...(reason ? { dropped: [{ field: 'changes.pr_url', reason }] } : {}) } }); + const run = await state(config, runId); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(expected ? [expected] : undefined); + expect(JSON.stringify(run)).not.toContain('session='); + }); + + it('can cancel a persisted delivery without treating it as settled success', async () => { + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + const run = await state(config, runId); + run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; + await writeRun(config, run); + expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + }); + + it('walks undelivered origin branches and deduplicates URLs from passes and archives', async () => { + const { config, runId } = await fixture(` - id: write\n command: echo test\n`); + const run = await state(config, runId); + const changes = { branch: 'first', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', prRecordedAt: 2 }; + run.copies = { second: { owner: driver, origin: 'first' }, third: { owner: driver, origin: 'second' } }; + run.steps = [{ id: 'write', status: 'succeeded', outcome: 'success', settledAt: 4, changes: { ...changes, branch: 'third', prRecordedAt: 4 }, passes: [{ pass: 1, outcome: 'success', changes }] }]; + run.archive = [{ step: 'older', pass: 1, outcome: 'success', changes: { ...changes, pr_url: 'https://review.example/2', prRecordedAt: 3 } }]; + expect(deliveryTarget(run, 'third')).toBe('first'); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://review.example/1', 'https://review.example/2']); + if (run.steps[0]?.status === 'succeeded') run.steps[0].outputs = { pr_url: 'https://untrusted.example/no' }; + expect(buildEndRecord(run, driver, 'tester').prs).not.toContain('https://untrusted.example/no'); + }); + + it('opens a separate PR per new-workspace loop pass and retains all earlier URLs', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls.map(([opts]) => opts.source)).toEqual([1, 2, 3].map((pass) => `teamai/${runId}/write/p${pass}`)); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual([1, 2, 3].map((n) => `https://github.com/fixture/app/pull/${n}`)); + }); + + it('reuses one PR when a loop continues the same branch', async () => { + const api = provider(); + api.lookup.mockImplementation(async () => api.create.mock.calls.length ? 'https://github.com/fixture/app/pull/1' : undefined); + const { config, cwd, runId } = await fixture(` - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n workspace: continue\n command: node -e "require('fs').appendFileSync('change.txt','pass\\n')"\n deliver: pr\n loop: { back_to: write, while: "length(steps.write.passes) < 3", max_iterations: 3 }\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create).toHaveBeenCalledTimes(1); expect(api.lookup).toHaveBeenCalledTimes(3); + expect(buildEndRecord(await state(config, runId), driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('passes over a skipped first need to the earlier delivered branch', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: first\n command: ${command}\n workspace: new\n deliver: pr\n - id: other\n needs: []\n workspace: new\n command: node -e "require('fs').writeFileSync('other.txt','other')"\n deliver: pr\n - id: skipped\n needs: [first]\n if: steps.first.outcome == 'failed'\n command: echo skipped\n - id: last\n needs: [skipped, other]\n workspace: new\n command: node -e "require('fs').writeFileSync('last.txt','last')"\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd), JSON.stringify(await state(config, runId))).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(api.create.mock.calls[2]?.[0]).toMatchObject({ source: `teamai/${runId}/last/p1`, target: `teamai/${runId}/first/p1` }); + }); + + it('rejects inherited protected changes even when the delivering step did not edit those paths', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: earlier\n workspace: new\n command: node -e "require('fs').mkdirSync('.github/workflows',{recursive:true});require('fs').writeFileSync('.github/workflows/ci.yml','bad')"\n - id: later\n needs: [earlier]\n workspace: new\n command: ${command}\n deliver: pr\n`); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ failure: { message: expect.stringContaining('.github/workflows/ci.yml') } }); + expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); + }); + + it('allows cancellation during provider I/O and retains the resulting PR receipt without reviving the step', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + api.create.mockImplementation(async () => { + expect(await cancelRun(config, runId, 'write')).toMatchObject({ data: { run: { status: 'failed' } } }); + return 'https://github.com/fixture/app/pull/99'; + }); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'cancelled', changes: { pr_url: 'https://github.com/fixture/app/pull/99' } }); + expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/99']); + }); + + it('keeps an execution deadline across delivery and makes no provider call once it expired', async () => { + const api = provider(); + const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + const run = await state(config, runId); + run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, deliveryDeadline: Date.now() - 1, + changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [] } }]; + await writeRun(config, run); + expect(await leaseNext(config, runId, driver)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { code: 'timeout' } }); + expect(api.lookup).not.toHaveBeenCalled(); expect(api.create).not.toHaveBeenCalled(); + }); + + it.each(['https://oauth2:fixture-token@github.com/fixture/app.git', 'ssh://git@github.com:22/fixture/app.git', 'git@github.com:fixture/app.git'])('recognizes hosting metadata through %s without passing transport credentials', async (remote) => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: echo test\n`); + git(cwd, ['remote', 'set-url', 'origin', remote]); + expect(await validateReportedPr(await state(config, runId), { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ url: 'https://github.com/fixture/app/pull/1' }); + expect(api.validate).toHaveBeenCalledWith('https://github.com/fixture/app/pull/1', expect.objectContaining({ repo: 'fixture/app' })); + expect(JSON.stringify(api.validate.mock.calls)).not.toContain('fixture-token'); + }); +}); diff --git a/src/__tests__/workflow-docs-examples.test.ts b/src/__tests__/workflow-docs-examples.test.ts index fca03bc4f..2fd399e24 100644 --- a/src/__tests__/workflow-docs-examples.test.ts +++ b/src/__tests__/workflow-docs-examples.test.ts @@ -75,6 +75,8 @@ function startsOf(page: string): { starts: Start[]; unread: string[] } { const unread = [...text.matchAll(/--inputs[ =]['"{]/g)] .map((match) => lineOf(match.index)) .filter((line) => !starts.some((start) => start.line === line)) + // Actions expressions become JSON only inside the job; the documented CLI path has its own real-CLI E2E. + .filter((line) => !text.split('\n')[line - 1]?.includes('${{')) .map((line) => `${page}:${line}: --inputs is not part of a teamai workflow start line`); return { starts, unread }; } diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts new file mode 100644 index 000000000..311b48532 --- /dev/null +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as github from '../providers/github/gh-cli.js'; +import * as gitcode from '../providers/gitcode/gitcode-api.js'; +import * as tgit from '../providers/tgit/rest-auth.js'; +import * as cnb from '../providers/cnb/cnb-cli.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import { GitLabProvider } from '../providers/gitlab/index.js'; +import { GitCodeProvider } from '../providers/gitcode/index.js'; +import { TGitProvider } from '../providers/tgit/index.js'; +import { CNBProvider } from '../providers/cnb/index.js'; +import type { PrCreateOptions } from '../providers/types.js'; + +const opts: PrCreateOptions = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', title: 'Change', spawnTimeoutMs: 1000 }; +afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); + +describe('workflow provider PR reuse and validation', () => { + it('reuses the exact GitHub repository/head/base and never accepts a fork with the same branch name', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + const pull = { state: 'open', html_url: 'https://github.com/acme/app/pull/7', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: 'main', repo: { full_name: opts.repo } } }; + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([{ ...pull, head: { ref: opts.source, repo: { full_name: 'fork/app' } }, html_url: 'https://github.com/acme/app/pull/6' }, pull]))); + vi.stubGlobal('fetch', fetcher); + const provider = new GitHubProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe(pull.html_url); + const url = new URL(fetcher.mock.calls[0]![0] as string); + expect(url.searchParams.get('head')).toBe(`acme:${opts.source}`); + expect(url.searchParams.get('base')).toBe('main'); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify(pull))); + expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(true); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...pull, base: { ...pull.base, ref: 'other' } }))); + expect(await provider.validatePullRequest(pull.html_url, opts)).toBe(false); + expect(await provider.validatePullRequest('https://github.com/other/app/pull/7', opts)).toBe(false); + const calls = fetcher.mock.calls.length; + expect(await provider.validatePullRequest('https://private-review.example/1', opts)).toBeUndefined(); + expect(fetcher).toHaveBeenCalledTimes(calls); + }); + + it('uses an already authenticated gh without invoking login', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(true); + const cli = vi.spyOn(github, 'ghExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([{ state: 'open', html_url: 'https://github.com/acme/app/pull/8', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }]) }); + expect(await new GitHubProvider().findOpenPullRequest(opts)).toBe('https://github.com/acme/app/pull/8'); + expect(cli.mock.calls[0]?.[0][0]).toBe('api'); + expect(cli.mock.calls.flatMap(([args]) => args)).not.toContain('login'); + }); + + it('fails noninteractively without GitHub credentials, without echoing response bodies', async () => { + vi.spyOn(github, 'isGhInstalled').mockReturnValue(false); + vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); + await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub authentication unavailable'); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('private-token-in-body', { status: 403 }))); + await expect(new GitHubProvider().findOpenPullRequest(opts)).rejects.toThrow('GitHub PR lookup failed (403). Check token and repository access.'); + }); + + it('validates GitLab project and branches at the configured instance, including a subpath', async () => { + vi.stubEnv('GITLAB_TOKEN', 'test-token'); vi.stubEnv('GITLAB_URL', 'https://gitlab.example/team'); + const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 5, target_project_id: 5, web_url: 'https://gitlab.example/team/acme/app/-/merge_requests/9' }; + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([mr]))); + vi.stubGlobal('fetch', fetcher); + const provider = new GitLabProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe(mr.web_url); + expect(String(fetcher.mock.calls[0]?.[0])).toContain('/team/api/v4/projects/acme%2Fapp/merge_requests?'); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify(mr))); + expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(true); + expect(await provider.validatePullRequest('https://gitlab.example/team/other/app/-/merge_requests/9', opts)).toBe(false); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify({ ...mr, source_project_id: 99 }))); + expect(await provider.validatePullRequest(mr.web_url, opts)).toBe(false); + }); + + it('reuses TGit requests through its existing REST authentication', async () => { + const mr = { state: 'opened', source_branch: opts.source, target_branch: opts.target, source_project_id: 4, target_project_id: 4, web_url: 'https://git.woa.com/acme/app/merge_requests/3' }; + const fetcher = vi.spyOn(tgit, 'tgitFetch').mockResolvedValue(new Response(JSON.stringify([mr]))); + expect(await new TGitProvider().findOpenPullRequest(opts)).toBe(mr.web_url); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify([mr]))); + expect(await new TGitProvider().validatePullRequest(mr.web_url, opts)).toBe(true); + }); + + it('paginates GitCode open requests before deciding to create a new one', async () => { + vi.spyOn(gitcode, 'getGitCodeToken').mockReturnValue('test-token'); + const pull = { state: 'open', html_url: 'https://gitcode.com/acme/app/pull/2', head: { ref: opts.source, repo: { full_name: opts.repo } }, base: { ref: opts.target, repo: { full_name: opts.repo } } }; + const fetcher = vi.fn().mockResolvedValueOnce(new Response(JSON.stringify(Array.from({ length: 100 }, () => ({ ...pull, head: { ...pull.head, ref: 'other' } }))))) + .mockResolvedValueOnce(new Response(JSON.stringify([pull]))); + vi.stubGlobal('fetch', fetcher); + expect(await new GitCodeProvider().findOpenPullRequest(opts)).toBe(pull.html_url); + expect(String(fetcher.mock.calls[1]?.[0])).toContain('page=2'); + }); + + it('uses the CNB CLI schema and normalizes refs/heads branch names', async () => { + const pull = { state: 'open', number: '12', head: { ref: `refs/heads/${opts.source}`, repo: { path: opts.repo } }, base: { ref: 'refs/heads/main', repo: { path: opts.repo } } }; + const cli = vi.spyOn(cnb, 'cnbExec').mockReturnValue({ status: 0, stderr: '', stdout: JSON.stringify([pull]) }); + const provider = new CNBProvider(); + expect(await provider.findOpenPullRequest(opts)).toBe('https://cnb.cool/acme/app/-/pulls/12'); + expect(cli.mock.calls[0]?.[0]).toContain('list-pulls'); + cli.mockReturnValueOnce({ status: 0, stderr: '', stdout: JSON.stringify(pull) }); + expect(await provider.validatePullRequest('https://cnb.cool/acme/app/-/pulls/12', opts)).toBe(true); + }); +}); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index 196df3b60..8674c4c4c 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -204,8 +204,8 @@ describe('#1083 workflow run records', () => { it('publishes the failure code of every step failure teamai writes, and only codes it can name', () => { // Codes a module writes, as `code: '…'`, `error('…', …)` or `failure('…', …)`, that never become a run's failure: every module of src/workflow and src/scorers, and every other one that imports src/scorers or answers a WorkflowEnvelope. - const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'workflow_namespace_conflict', 'prompt_namespace_conflict'] satisfies Array; - const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'invalid_report', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'busy', 'workflow_recursion', 'workflow_not_found', + const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'workflow_namespace_conflict', 'prompt_namespace_conflict', 'bad_delivery'] satisfies Array; + const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'invalid_report', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'dropped', 'busy', 'workflow_recursion', 'workflow_not_found', // Refusals of a run handover, answered to the command or driver that asked for it. 'not_owner', 'unsupported_repo', 'reports_unreachable', 'reports_failed', 'run_not_found', 'run_not_published', 'run_mismatch', 'state_conflict', 'state_moved', 'definition_mismatch', 'records_pending', 'snapshot_unsupported', 'secret_detected', 'local_path_detected', 'origin_unavailable', 'commit_missing', diff --git a/src/dashboard/client.ts b/src/dashboard/client.ts index 11cba0357..bfce94e0e 100644 --- a/src/dashboard/client.ts +++ b/src/dashboard/client.ts @@ -188,7 +188,7 @@ async function saveSelection() { if(!dist||!distDraft||!preview)return; const ge // records the team published. Rows carry only run-record fields, and a HITL // answer posts the pass the page showed so a replaced request is refused. const runStatusNames = { running:'Running', succeeded:'Succeeded', failed:'Failed', cancelled:'Cancelled' }; -const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; +const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', delivering:'Delivering PR', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; function runStatus(status) { return ''+label(runStatusNames[status]||stepStatusNames[status]||status)+''; } // A running run's duration is filled by tickRunDurations, so a poll that changes nothing leaves the HTML, and the buttons in it, alone. function runDurationHtml(r) { return r.durationMs==null&&r.status==='running'?'':e(elapsed(r.durationMs ?? NaN)); } diff --git a/src/dashboard/locales.ts b/src/dashboard/locales.ts index a4174bdd3..411191896 100644 --- a/src/dashboard/locales.ts +++ b/src/dashboard/locales.ts @@ -264,6 +264,7 @@ export const dashboardMessages: Record = { "Cancelled": "已取消", "Pending": "待运行", "Leased": "已租用", + "Delivering PR": "正在交付 PR", "Skipped": "已跳过", "Waiting for an answer": "等待回答", "Waiting": "等待中", diff --git a/src/dashboard/workflow-runs.ts b/src/dashboard/workflow-runs.ts index 09b4d0cd3..1fd0917ee 100644 --- a/src/dashboard/workflow-runs.ts +++ b/src/dashboard/workflow-runs.ts @@ -92,7 +92,7 @@ interface RunRow { /** A run on this machine, as its run file stands now. */ export type LocalRunRow = RunRow & { source: 'local'; - /** The leased steps: a driver may work on several at once. */ + /** Agent leases and deliveries currently in progress. */ current: string[]; /** The steps whose HITL request waits for an answer. */ waiting: string[]; @@ -176,7 +176,7 @@ function localRow(run: RunState, records: Map): LocalRunRow return { ...rowOf(localRecord(run)), source: 'local', - current: run.steps.filter((state) => state.status === 'leased').map((state) => state.id), + current: run.steps.filter((state) => state.status === 'leased' || state.status === 'delivering').map((state) => state.id), waiting: run.steps.filter((state) => state.status === 'waiting').map((state) => state.id), ...(record ? { record } : {}), }; diff --git a/src/index.ts b/src/index.ts index af78faf6d..723d4a0af 100644 --- a/src/index.ts +++ b/src/index.ts @@ -795,7 +795,7 @@ envCmd const workflowCmd = program .command('workflow') - .description('Run a validated workflow from workflows/.yaml'); + .description('Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them.'); workflowCmd .command('start ') diff --git a/src/providers/cnb/cnb-cli.ts b/src/providers/cnb/cnb-cli.ts index dc85b8d00..43e24785c 100644 --- a/src/providers/cnb/cnb-cli.ts +++ b/src/providers/cnb/cnb-cli.ts @@ -43,7 +43,7 @@ export const CNB_HOST = process.env.TEAMAI_CNB_HOST?.trim() || 'cnb.cool'; /** Run a `cnb` subcommand. Returns { stdout, stderr, status }. */ export function cnbExec( args: string[], - options?: { inheritStdio?: boolean; cwd?: string }, + options?: { inheritStdio?: boolean; cwd?: string; timeoutMs?: number }, ): { stdout: string; stderr: string; status: number } { // Resolve the executable first, then launch it through cross-spawn: on // Windows the npm-installed CLI is only a `.cmd` shim, which neither a bare @@ -58,7 +58,7 @@ export function cnbExec( log.debug(`cnb exec: ${cnbPath} ${args.join(' ')}`); if (options?.inheritStdio) { - const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd }); + const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd, ...(options.timeoutMs ? { timeout: options.timeoutMs } : {}) }); return { stdout: '', stderr: '', status: r.status ?? 1 }; } const r = crossSpawn.sync(cnbPath, args, { @@ -66,6 +66,7 @@ export function cnbExec( encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024, cwd: options?.cwd, + ...(options?.timeoutMs ? { timeout: options.timeoutMs } : {}), }); return { stdout: (r.stdout ?? '').toString().trim(), @@ -339,6 +340,7 @@ export interface CnbPullCreateOptions { title: string; description?: string; cwd?: string; + spawnTimeoutMs?: number; } /** @@ -357,7 +359,7 @@ export function cnbPullCreate(opts: CnbPullCreateOptions): string { ]; if (opts.description) args.push('--body', opts.description); - const r = cnbExec(args, { cwd: opts.cwd }); + const r = cnbExec(args, { cwd: opts.cwd, ...(opts.spawnTimeoutMs ? { timeoutMs: opts.spawnTimeoutMs } : {}) }); if (r.status !== 0) { throw new Error(`cnb post-pull failed: ${r.stderr || r.stdout}`); } diff --git a/src/providers/cnb/index.ts b/src/providers/cnb/index.ts index b3a3a02a9..2eb62aa2e 100644 --- a/src/providers/cnb/index.ts +++ b/src/providers/cnb/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, RepoInfo, PrCreateOptions } from '../types.js'; +import { findCnbPull, validateCnbPull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { cnbParseRepoInput, @@ -19,6 +20,8 @@ import { * the TGit provider's "thin class over a platform CLI" shape. */ export class CNBProvider implements GitProvider { + findOpenPullRequest(opts: PrCreateOptions): Promise { return findCnbPull(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateCnbPull(url, opts); } readonly name = 'cnb'; parseRepoInput(input: string): RepoInfo { @@ -72,6 +75,7 @@ export class CNBProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/cnb/workflow-pr.ts b/src/providers/cnb/workflow-pr.ts new file mode 100644 index 000000000..23fde2e45 --- /dev/null +++ b/src/providers/cnb/workflow-pr.ts @@ -0,0 +1,41 @@ +import { parse } from 'yaml'; +import type { PrCreateOptions } from '../types.js'; +import { CNB_HOST, cnbExec } from './cnb-cli.js'; + +interface Repository { path?: string; web_url?: string } +interface Pull { number: string; state: string; head: { ref: string; repo?: Repository }; base: { ref: string; repo?: Repository } } +function matches(pull: Pull, opts: PrCreateOptions): boolean { + const ref = (value: string): string => value.replace(/^refs\/heads\//, ''); + const repoMatches = (repo: Repository | undefined): boolean => { + if (repo?.web_url) { + const url = new URL(repo.web_url); + return url.host === CNB_HOST && url.pathname.replace(/^\/|\/$/g, '') === opts.repo; + } + return !repo?.path || repo.path === opts.repo || repo.path === opts.repo.split('/').at(-1); + }; + return pull.state === 'open' && ref(pull.head.ref) === opts.source && ref(pull.base.ref) === opts.target + && repoMatches(pull.head.repo) && repoMatches(pull.base.repo); +} +function request(command: string, args: string[], opts: PrCreateOptions): unknown { + const response = cnbExec(['pulls', command, '--repo', opts.repo, ...args], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); + if (response.status !== 0) throw new Error('CNB PR lookup failed. Check cnb authentication or CNB_TOKEN and repository access.'); + try { return parse(response.stdout) as unknown; } + catch { throw new Error('CNB PR lookup returned invalid data. Inspect cnb before retrying.'); } +} +export async function findCnbPull(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const pulls = request('list-pulls', ['--state', 'open', '--base-ref', opts.target, '--page-size', '100', '--page', String(page)], opts) as Pull[]; + if (!Array.isArray(pulls)) throw new Error('CNB PR lookup returned no request list. Inspect cnb before retrying.'); + const found = pulls.find((pull) => matches(pull, opts)); + if (found) return `https://${CNB_HOST}/${opts.repo}/-/pulls/${found.number}`; + if (pulls.length < 100) return undefined; + } + throw new Error('Too many open CNB requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateCnbPull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.host !== CNB_HOST) return undefined; + const prefix = `/${opts.repo}/-/pulls/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(request('get-pull', ['--number', url.pathname.slice(prefix.length).replace(/\/$/, '')], opts) as Pull, opts); +} diff --git a/src/providers/gitcode/gitcode-api.ts b/src/providers/gitcode/gitcode-api.ts index b4ea6da60..f51c155d0 100644 --- a/src/providers/gitcode/gitcode-api.ts +++ b/src/providers/gitcode/gitcode-api.ts @@ -274,6 +274,7 @@ export interface GitCodePullCreateOptions { description?: string; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } /** @@ -296,6 +297,7 @@ export async function gitcodePullCreate(opts: GitCodePullCreateOptions): Promise headers: authHeaders(token), body: JSON.stringify(body), redirect: 'manual', + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), }); if (!resp.ok) { const errBody = await resp.text().catch(() => ''); diff --git a/src/providers/gitcode/index.ts b/src/providers/gitcode/index.ts index 88ec983d2..e710844a8 100644 --- a/src/providers/gitcode/index.ts +++ b/src/providers/gitcode/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import { findGitCodePull, validateGitCodePull } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { gitcodeIsAuthenticated, @@ -25,6 +26,8 @@ import { askQuestion, isInteractive } from '../../utils/prompt.js'; * that distinguish it from the GitLab provider it is structurally modelled on. */ export class GitCodeProvider implements GitProvider { + findOpenPullRequest(opts: PrCreateOptions): Promise { return findGitCodePull(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateGitCodePull(url, opts); } readonly name = 'gitcode'; parseRepoInput(input: string): RepoInfo { @@ -111,6 +114,7 @@ export class GitCodeProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/gitcode/workflow-pr.ts b/src/providers/gitcode/workflow-pr.ts new file mode 100644 index 000000000..675337d82 --- /dev/null +++ b/src/providers/gitcode/workflow-pr.ts @@ -0,0 +1,37 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitCodeToken, gitcodeApiBase } from './gitcode-api.js'; +import { GITCODE_HOST } from './repo-url.js'; + +interface Pull { html_url: string; state: string; head: { ref: string; repo?: { full_name?: string } }; base: { ref: string; repo?: { full_name?: string } } } +function matches(pull: Pull, opts: PrCreateOptions): boolean { + return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target + && (!pull.head.repo?.full_name || pull.head.repo.full_name === opts.repo) + && (!pull.base.repo?.full_name || pull.base.repo.full_name === opts.repo); +} +async function request(route: string, opts: PrCreateOptions): Promise { + const token = getGitCodeToken(); + if (!token) throw new Error('GitCode authentication unavailable. Set GITCODE_TOKEN before resuming.'); + const response = await fetch(`${gitcodeApiBase()}/repos/${opts.repo}/pulls${route}`, { + headers: { Authorization: `Bearer ${token}`, Accept: 'application/json' }, redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitCode PR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('gitcode review lookup returned invalid data.'); }); +} +export async function findGitCodePull(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const query = new URLSearchParams({ state: 'open', base: opts.target, per_page: '100', page: String(page) }); + const pulls = await request(`?${query}`, opts) as Pull[]; + const found = pulls.find((pull) => matches(pull, opts)); + if (found) return found.html_url; + if (pulls.length < 100) return undefined; + } + throw new Error('Too many open GitCode requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateGitCodePull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.hostname !== GITCODE_HOST) return undefined; + const prefix = `/${opts.repo}/pull/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as Pull, opts); +} diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index e7a1c49c4..8ce343338 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -562,6 +562,7 @@ async function ghPrCreateViaApi(opts: GhPrCreateOptions): Promise { { method: 'POST', headers: authHeaders, + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), body: JSON.stringify({ title: opts.title, body: opts.description ?? opts.title, diff --git a/src/providers/github/index.ts b/src/providers/github/index.ts index 9d7af0342..b24c3fbf7 100644 --- a/src/providers/github/index.ts +++ b/src/providers/github/index.ts @@ -12,6 +12,7 @@ import { } from './gh-cli.js'; import { ghListOrgRepos } from './gh-org.js'; import { parseGitHubRepoInput } from './repo-url.js'; +import { findGitHubPull, validateGitHubPull } from './workflow-pr.js'; export class GitHubProvider implements GitProvider { readonly name = 'github'; @@ -64,6 +65,14 @@ export class GitHubProvider implements GitProvider { }); } + findOpenPullRequest(opts: PrCreateOptions): Promise { + return findGitHubPull(opts); + } + + validatePullRequest(url: string, opts: PrCreateOptions): Promise { + return validateGitHubPull(url, opts); + } + getDefaultEmailDomain(): string | null { return null; } diff --git a/src/providers/github/workflow-pr.ts b/src/providers/github/workflow-pr.ts new file mode 100644 index 000000000..dde389cd4 --- /dev/null +++ b/src/providers/github/workflow-pr.ts @@ -0,0 +1,48 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitHubToken, ghExec, isGhInstalled } from './gh-cli.js'; + +interface Pull { + html_url: string; + state: string; + head: { ref: string; repo: { full_name: string } | null }; + base: { ref: string; repo: { full_name: string } }; +} + +async function request(path: string, opts: PrCreateOptions): Promise { + if (isGhInstalled()) { + const result = ghExec(['api', path], { cwd: opts.cwd, timeoutMs: opts.spawnTimeoutMs ?? 30_000 }); + if (result.status !== 0) throw new Error('GitHub PR lookup failed. Check gh authentication and repository access.'); + try { return JSON.parse(result.stdout) as unknown; } + catch { throw new Error('GitHub PR lookup returned invalid data. Inspect gh before retrying.'); } + } + const token = getGitHubToken(); + if (!token) throw new Error('GitHub authentication unavailable. Set GITHUB_TOKEN or authenticate gh before resuming.'); + const response = await fetch(`https://api.github.com/${path}`, { + headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, + redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitHub PR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('GitHub PR lookup returned invalid data.'); }); +} + +function matches(pull: Pull, opts: PrCreateOptions): boolean { + return pull.state === 'open' && pull.head.ref === opts.source && pull.base.ref === opts.target + && pull.head.repo?.full_name.toLowerCase() === opts.repo.toLowerCase() + && pull.base.repo.full_name.toLowerCase() === opts.repo.toLowerCase(); +} + +export async function findGitHubPull(opts: PrCreateOptions): Promise { + const query = new URLSearchParams({ state: 'open', head: `${opts.repo.split('/')[0]}:${opts.source}`, base: opts.target, per_page: '100' }); + const pulls = await request(`repos/${opts.repo}/pulls?${query}`, opts) as Pull[]; + return pulls.find((pull) => matches(pull, opts))?.html_url; +} + +export async function validateGitHubPull(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + // A remote driver may use a separate review service. Validate recognized provider URLs only. + if (url.hostname !== 'github.com') return undefined; + const match = /^\/([^/]+\/[^/]+)\/pull\/(\d+)\/?$/.exec(url.pathname); + if (!match || match[1]!.toLowerCase() !== opts.repo.toLowerCase()) return false; + return matches(await request(`repos/${opts.repo}/pulls/${match[2]}`, opts) as Pull, opts); +} diff --git a/src/providers/gitlab/gitlab-api.ts b/src/providers/gitlab/gitlab-api.ts index 9409bf0d5..a2a426cc4 100644 --- a/src/providers/gitlab/gitlab-api.ts +++ b/src/providers/gitlab/gitlab-api.ts @@ -301,6 +301,7 @@ export interface GitLabMrCreateOptions { reviewers?: string[]; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } interface GitLabUser { @@ -355,6 +356,7 @@ export async function gitlabMrCreate(opts: GitLabMrCreateOptions): Promise ''); diff --git a/src/providers/gitlab/index.ts b/src/providers/gitlab/index.ts index 506a810a5..5d05e8f90 100644 --- a/src/providers/gitlab/index.ts +++ b/src/providers/gitlab/index.ts @@ -13,6 +13,7 @@ import { gitlabListOrgRepos } from './org.js'; import { fetchGitLabMR } from './mr-fetch.js'; import { parseGitLabRepoInput } from './repo-url.js'; import type { MRData } from '../../types.js'; +import { findGitLabMergeRequest, validateGitLabMergeRequest } from './workflow-pr.js'; /** * GitLab provider (supports self-hosted / enterprise GitLab instances). @@ -72,6 +73,7 @@ export class GitLabProvider implements GitProvider { description: opts.description, reviewers: opts.reviewers, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } @@ -79,6 +81,14 @@ export class GitLabProvider implements GitProvider { return fetchGitLabMR(url); } + findOpenPullRequest(opts: PrCreateOptions): Promise { + return findGitLabMergeRequest(opts); + } + + validatePullRequest(url: string, opts: PrCreateOptions): Promise { + return validateGitLabMergeRequest(url, opts); + } + async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gitlabListOrgRepos(org, opts); } diff --git a/src/providers/gitlab/workflow-pr.ts b/src/providers/gitlab/workflow-pr.ts new file mode 100644 index 000000000..cf8c1ed9a --- /dev/null +++ b/src/providers/gitlab/workflow-pr.ts @@ -0,0 +1,42 @@ +import type { PrCreateOptions } from '../types.js'; +import { getGitLabToken, gitlabApiBase, gitlabBaseUrl } from './gitlab-api.js'; + +interface MergeRequest { + web_url: string; + state: string; + source_branch: string; + target_branch: string; + source_project_id: number; + target_project_id: number; +} + +function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { + return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target + && mr.source_project_id === mr.target_project_id; +} + +async function request(route: string, opts: PrCreateOptions): Promise { + const token = getGitLabToken(); + if (!token) throw new Error('GitLab authentication unavailable. Set GITLAB_TOKEN before resuming.'); + const response = await fetch(`${gitlabApiBase()}/projects/${encodeURIComponent(opts.repo)}/merge_requests${route}`, { + headers: { 'PRIVATE-TOKEN': token, Accept: 'application/json' }, redirect: 'manual', + signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`GitLab MR lookup failed (${response.status}). Check token and repository access.`); + return response.json().catch(() => { throw new Error('gitlab review lookup returned invalid data.'); }); +} + +export async function findGitLabMergeRequest(opts: PrCreateOptions): Promise { + const query = new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100' }); + const requests = await request(`?${query}`, opts) as MergeRequest[]; + return requests.find((mr) => matches(mr, opts))?.web_url; +} + +export async function validateGitLabMergeRequest(value: string, opts: PrCreateOptions): Promise { + const base = new URL(gitlabBaseUrl()); + const url = new URL(value); + if (url.origin !== base.origin) return undefined; + const prefix = `${base.pathname.replace(/\/$/, '')}/${opts.repo}/-/merge_requests/`; + if (!url.pathname.startsWith(prefix) || !/^\d+\/?$/.test(url.pathname.slice(prefix.length))) return false; + return matches(await request(`/${url.pathname.slice(prefix.length).replace(/\/$/, '')}`, opts) as MergeRequest, opts); +} diff --git a/src/providers/tgit/index.ts b/src/providers/tgit/index.ts index da5d724f0..ddc583617 100644 --- a/src/providers/tgit/index.ts +++ b/src/providers/tgit/index.ts @@ -1,4 +1,5 @@ import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import { findTGitMergeRequest, validateTGitMergeRequest } from './workflow-pr.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGfInstalled, @@ -68,6 +69,9 @@ export class TGitProvider implements GitProvider { return 'tencent.com'; } + findOpenPullRequest(opts: PrCreateOptions): Promise { return findTGitMergeRequest(opts); } + validatePullRequest(url: string, opts: PrCreateOptions): Promise { return validateTGitMergeRequest(url, opts); } + async listOrgRepos(org: string, opts?: { maxRepos?: number }): Promise { return gfListOrgRepos(org, opts); } diff --git a/src/providers/tgit/workflow-pr.ts b/src/providers/tgit/workflow-pr.ts new file mode 100644 index 000000000..96b7c7c36 --- /dev/null +++ b/src/providers/tgit/workflow-pr.ts @@ -0,0 +1,32 @@ +import type { PrCreateOptions } from '../types.js'; +import { tgitFetch } from './rest-auth.js'; + +interface MergeRequest { web_url: string; state: string; source_branch: string; target_branch: string; source_project_id: number; target_project_id: number } +function matches(mr: MergeRequest, opts: PrCreateOptions): boolean { + return mr.state === 'opened' && mr.source_branch === opts.source && mr.target_branch === opts.target + && mr.source_project_id === mr.target_project_id; +} +async function list(opts: PrCreateOptions, query: URLSearchParams): Promise { + const response = await tgitFetch(`/projects/${encodeURIComponent(opts.repo)}/merge_requests?${query}`, { + redirect: 'manual', signal: AbortSignal.timeout(opts.spawnTimeoutMs ?? 30_000), + }); + if (!response.ok) throw new Error(`TGit MR lookup failed (${response.status}). Check TGIT_TOKEN or gf authentication.`); + return response.json().catch(() => { throw new Error('TGit MR lookup returned invalid data.'); }) as Promise; +} +export async function findTGitMergeRequest(opts: PrCreateOptions): Promise { + for (let page = 1; page <= 20; page += 1) { + const requests = await list(opts, new URLSearchParams({ state: 'opened', source_branch: opts.source, target_branch: opts.target, per_page: '100', page: String(page) })); + const found = requests.find((mr) => matches(mr, opts)); + if (found) return found.web_url; + if (requests.length < 100) return undefined; + } + throw new Error('Too many open TGit requests to establish whether this branch already has one. Inspect it manually.'); +} +export async function validateTGitMergeRequest(value: string, opts: PrCreateOptions): Promise { + const url = new URL(value); + if (url.hostname !== 'git.woa.com') return undefined; + const match = /^\/(.+?)\/(?:-\/)?merge_requests\/(\d+)\/?$/.exec(url.pathname); + if (!match || match[1] !== opts.repo) return false; + const requests = await list(opts, new URLSearchParams({ iid: match[2]! })); + return requests.length === 1 && matches(requests[0]!, opts); +} diff --git a/src/providers/types.ts b/src/providers/types.ts index ccf95ab8b..4b2bae05d 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -120,6 +120,12 @@ export interface GitProvider { */ createPullRequest(opts: PrCreateOptions): Promise; + /** Finds an already open request, so a delivery restarted after a crash does not create another one. */ + findOpenPullRequest?(opts: PrCreateOptions): Promise; + + /** False means known metadata disagrees; undefined means this provider cannot validate the URL. */ + validatePullRequest?(url: string, opts: PrCreateOptions): Promise; + /** * 获取指定 MR/PR 的完整数据（标题、描述、提交列表、diff）。 * diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index dd94ac917..a56819c80 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -28,7 +28,7 @@ export interface ConformanceScenario { * `absent` lists fields the lease must not carry. */ export type ConformanceOperation = - | { kind: 'next'; saveAs: string; running?: string[]; canTakeLease?: boolean; hitl?: boolean; claim?: boolean; cancelled?: string[]; absent?: string[]; expect: Record } + | { kind: 'next'; saveAs: string; running?: string[]; canTakeLease?: boolean; hitl?: boolean; claim?: boolean; workspace?: boolean; cancelled?: string[]; absent?: string[]; expect: Record } | { kind: 'respond'; step: string; choice: string; note?: string; pass?: number; json?: unknown; expect: Record } | { kind: 'retry'; step: string; expect: Record } | { kind: 'cancel'; step?: string; expect: Record } @@ -37,6 +37,8 @@ export type ConformanceOperation = lease?: string; attempt?: string; revision?: number; + /** The harness fills the changes branch/base from this operation's saved lease. */ + leasedChanges?: boolean; body: Record; expect: Record; }; @@ -54,6 +56,15 @@ steps: ok: boolean `; +const REVIEW_LINK = `schemaVersion: v1alpha1 +name: review-link +description: A remote driver reports a review link. +steps: + - id: work + workspace: new + command: echo work +`; + const FAILED_DEPENDENCY = `schemaVersion: v1alpha1 name: build description: A step whose dependency failed is skipped. @@ -240,6 +251,23 @@ steps: `; export const workflowConformanceScenarios: ConformanceScenario[] = [ + { + name: 'a driver reports a sanitized review URL', workflowName: 'review-link', yaml: REVIEW_LINK, + operations: [ + { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' } }, expect: { data: { accepted: true, ignored: ['changes.unknown'] } } }, + { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, + ], + expectState: { steps: [{ id: 'work', changes: { pr_url: 'https://review.example/changes/1' } }] }, + }, + { + name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, + operations: [ + { kind: 'next', saveAs: 'work', workspace: true, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', leasedChanges: true, body: { outcome: 'success', changes: { pr_url: 'https://user:token@review.example/changes/1' } }, expect: { data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } } }, + { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, + ], + }, { name: 'two-step command success', workflowName: 'build', diff --git a/src/workflow/delivery.ts b/src/workflow/delivery.ts new file mode 100644 index 000000000..2884f7070 --- /dev/null +++ b/src/workflow/delivery.ts @@ -0,0 +1,192 @@ +/** PR delivery is resumable I/O, separate from the agent lease and never under the run lock. */ +import path from 'node:path'; +import type { LocalConfig } from '../types.js'; +import type { GitProvider, PrCreateOptions } from '../providers/types.js'; +import { getProvider, getProviderFromUrl } from '../providers/registry.js'; +import { acquireLock, releaseLock } from '../update.js'; +import { fail, succeed, type StepState } from './step-state.js'; +import { readRun, runLockPath } from './store.js'; +import { withRun, ok } from './engine.js'; +import { sanitizePrUrl } from './pr-url.js'; +import { AGENT_PATHS, git } from './worktree.js'; +import type { StepChanges } from './workspace.js'; +import type { RunState } from './types.js'; +import { publishRunRecords, queueRunRecord } from './records.js'; + +/** Changes in earlier passes remain candidates for a stacked target. */ +function allChanges(run: RunState): StepChanges[] { + return [ + ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), + ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), + ]; +} + +/** Follows the actual copy origin, including after-item precedence, through branches that did not deliver. */ +export function deliveryTarget(run: RunState, branch: string): string | undefined { + const changes = allChanges(run); + const visited = new Set([branch]); + let origin = run.copies?.[branch]?.origin; + while (origin && !visited.has(origin)) { + visited.add(origin); + if (changes.some((item) => item.branch === origin && item.pr_url)) return origin; + origin = run.copies?.[origin]?.origin; + } + return run.base?.ref; +} + +const CI_PATHS = ['.github/workflows', '.github/actions', '.gitlab-ci.yml', '.gitlab', '.coding-ci.yaml', '.cnb', '.circleci', '.buildkite', '.husky', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml']; + +export function protectedDeliveryPaths(files: string[]): string[] { + const roots = [...AGENT_PATHS, ...CI_PATHS].map((root) => root.toLowerCase()); + return files.filter((file) => { + const lower = file.toLowerCase(); + return /(^|\/)(agents|claude)\.md$/.test(lower) + || roots.some((root) => lower === root || lower.startsWith(`${root}/`)); + }); +} + +async function providerFor(run: RunState): Promise<{ provider: GitProvider; opts: PrCreateOptions }> { + if (!run.checkout) throw new Error('The run has no product checkout. Resume from a checkout that can access the delivered branch.'); + // Read the configured URL: `remote get-url` expands insteadOf transport rewrites and can hide the hosting provider. + const remote = await git(run.checkout, ['config', '--get', 'remote.origin.url']); + if (remote.code !== 0) throw new Error('The product checkout has no origin remote. Configure origin before resuming.'); + const url = remote.stdout.trim(); + let hostingUrl = url; + if (/^(https?|ssh):\/\//.test(url)) { + const parsed = new URL(url); + parsed.username = ''; parsed.password = ''; parsed.search = ''; parsed.hash = ''; + hostingUrl = parsed.protocol === 'ssh:' ? `https://${parsed.hostname}${parsed.pathname}` : parsed.toString(); + } else { + const scp = /^[^@\s]+@([^:\s]+):(.+)$/.exec(url); + if (scp) hostingUrl = `https://${scp[1]}/${scp[2]}`; + } + // Filesystem remotes are transport-only too; the registry's bare owner/repo fallback is for user input. + const provider = /^(https?:\/\/|ssh:\/\/|[^@\s]+@[^:\s]+:)/.test(url) ? getProviderFromUrl(hostingUrl) : getProvider('git'); + const repo = provider.name === 'git' ? hostingUrl : (() => { + const parsed = provider.parseRepoInput(hostingUrl); + return `${parsed.owner}/${parsed.repo}`; + })(); + return { provider, opts: { repo, source: '', target: '', title: '', cwd: run.checkout, spawnTimeoutMs: 30_000 } }; +} + +/** Report validation never echoes a rejected value, including a provider's error response. */ +export async function validateReportedPr(run: RunState, changes: StepChanges): Promise<{ url?: string; reason?: string }> { + const clean = sanitizePrUrl(changes.pr_url); + if (!clean.url) return clean; + if (!run.checkout) return clean; + try { + const { provider, opts } = await providerFor(run); + const target = deliveryTarget(run, changes.branch); + if (!target) return { reason: 'run base has no branch' }; + const valid = await provider.validatePullRequest?.(clean.url, { ...opts, source: changes.branch, target }); + return valid === false ? { reason: 'repository or branches do not match the lease' } : clean; + } catch { + return { reason: 'provider metadata could not be validated' }; + } +} + +async function createDelivery(run: RunState, state: Extract): Promise { + const remaining = (): number => { + const ms = state.deliveryDeadline === undefined ? 30_000 : Math.min(30_000, state.deliveryDeadline - Date.now()); + if (ms <= 0) throw new Error('The execution deadline expired before delivery completed.'); + return ms; + }; + remaining(); + const changes = state.changes; + if (!changes || !run.base || !run.checkout) throw new Error('Delivery needs a writable branch, a run base, and its product checkout.'); + const target = deliveryTarget(run, changes.branch); + if (!target) throw new Error('Run base has no branch. Start the run from a named branch (CI checkout ref: main).'); + if (![changes.to, run.base.commit].every((commit) => /^[0-9a-f]{40,64}$/i.test(commit))) throw new Error('Delivery requires full commit object ids.'); + // Do not inspect a later branch head or trust a driver's claimed list of commits. + let head = await git(run.checkout, ['rev-parse', '--verify', `refs/heads/${changes.branch}^{commit}`]); + if (head.code !== 0 || head.stdout.trim() !== changes.to) { + const fetched = await git(run.checkout, ['fetch', '--no-tags', 'origin', `refs/heads/${changes.branch}`]); + head = await git(run.checkout, ['rev-parse', '--verify', 'FETCH_HEAD^{commit}']); + if (fetched.code !== 0 || head.code !== 0 || head.stdout.trim() !== changes.to) throw new Error('The reported branch head is not available. Push the reported commit before resuming.'); + } + const diff = await git(run.checkout, ['diff', '--no-renames', '--name-only', '-z', run.base.commit, changes.to, '--']); + if (diff.code !== 0) throw new Error('Cannot inspect the delivered diff against the run base. Fetch the base commit before resuming.'); + const protectedPaths = protectedDeliveryPaths(diff.stdout.split('\0').filter(Boolean)); + if (protectedPaths.length) throw new Error(`Protected paths changed against the run base: ${protectedPaths.join(', ')}`); + const { provider, opts } = await providerFor(run); + opts.spawnTimeoutMs = remaining(); + if (changes.pr_url) return changes.pr_url; + const request: PrCreateOptions = { ...opts, source: changes.branch, target, + title: `${run.name}: ${state.id}`, description: `Workflow ${run.name}, run ${run.id}, step ${state.id}.\n\nBase: ${run.base.commit}\n\nCreated for human review. TeamAI never merges this request.` }; + const existing = await provider.findOpenPullRequest?.(request); + request.spawnTimeoutMs = remaining(); + let url = existing; + if (!url) { + try { url = await provider.createPullRequest(request); } + catch (error) { + // A provider's response body or CLI output may contain credentials. Keep its established transport-only + // fallback, but never persist arbitrary creation errors from authenticated providers. + if (provider.name === 'git') throw error; + throw new Error(`The ${provider.name} provider could not create the PR. Check its configured credentials, repository/branch access and push permissions, then retry delivery.`); + } + } + const clean = sanitizePrUrl(url); + if (!clean.url) throw new Error('Provider returned an unsafe PR URL. The branch is preserved; inspect the provider before retrying.'); + return clean.url; +} + +/** Drains durable delivery work once. A competing delivery process leaves it for the next poll. */ +export async function deliverPending(config: LocalConfig, runId: string): Promise { + const loaded = await readRun(config, runId); + if (typeof loaded === 'string' || loaded.status !== 'running' || !loaded.steps.some((step) => step.status === 'delivering')) return false; + const lock = path.join(path.dirname(runLockPath(config, runId)), `${runId}.delivery.lock`); + if (!await acquireLock(lock)) return false; + let changed = false; + let queued = false; + try { + for (;;) { + const run = await readRun(config, runId); + if (typeof run === 'string' || run.status !== 'running') break; + const state = run.steps.find((step): step is Extract => step.status === 'delivering'); + if (!state) break; + let url: string | undefined; + let failure: string | undefined; + try { url = await createDelivery(run, state); } + catch (error) { failure = error instanceof Error ? error.message : 'PR delivery failed'; } + const applied = await withRun(config, runId, async (current) => { + const step = current.steps.find((item) => item.id === state.id); + // Cancel/retry may have invalidated this delivery while the provider was running. + if (current.status !== 'running' || step?.status !== 'delivering' || step.pass !== state.pass || step.settledAt !== state.settledAt) { + // An in-flight provider request cannot be revoked. Keep its safe receipt on the matching completed + // attempt/pass, without resurrecting cancelled work, so the end record can still account for the PR. + if (!url || !state.changes) return { save: false, envelope: ok({}) }; + const candidates = [...(step && 'changes' in step ? [step] : []), ...(step?.passes ?? []), + ...(current.archive ?? []).filter((pass) => pass.step === state.id)]; + const receipt = candidates.find((pass) => pass.changes?.branch === state.changes!.branch && pass.changes.to === state.changes!.to + && pass.attempt === state.attempt && (pass.pass ?? 1) === (state.pass ?? 1)); + if (!receipt?.changes) return { save: false, envelope: ok({}) }; + current.revision += 1; + receipt.changes.pr_url = url; + receipt.changes.prRecordedAt ??= current.revision; + if (current.status !== 'running') queued = await queueRunRecord(config, current, current.driver, 'end') || queued; + changed = true; + return { save: true, envelope: ok({}) }; + } + current.revision += 1; + const { status: _status, ...settlement } = step; + const expired = step.deliveryDeadline !== undefined && Date.now() >= step.deliveryDeadline; + if (failure || expired) { + const message = `PR delivery failed for branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved. ${failure}`; + const detail = expired ? `Execution deadline expired while delivering branch ${step.changes?.branch ?? '(unknown)'}. Work is preserved.` : message; + fail(step, { ...settlement, settledAt: current.revision, failure: { code: expired ? 'timeout' : 'delivery_failed', message: detail }, message: detail, + ...(url && step.changes ? { changes: { ...step.changes, pr_url: url, prRecordedAt: step.changes.prRecordedAt ?? current.revision } } : {}) }); + } else { + succeed(step, { ...settlement, settledAt: current.revision, + changes: { ...step.changes!, pr_url: url!, prRecordedAt: step.changes?.prRecordedAt ?? current.revision } }); + } + changed = true; + return { save: true, envelope: ok({}) }; + }); + if ('error' in applied) break; + } + return changed; + } finally { + await releaseLock(lock); + if (queued) await publishRunRecords(config); + } +} diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 23b71a3bd..7f3a8de8a 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -18,16 +18,17 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - ReadSchema, answerHitl, cancel, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, + ReadSchema, answerHitl, cancel, deliver, fail, lease, release, settled, skip, start, stopped, succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; -import { generateRunId, installOwner, isRunId, updateRun } from './store.js'; +import { branchName, planWorkspace, plannedBranch, sessionOf, type StepChanges, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspaceMode, type WorkspacePlan } from './workspace.js'; +import { generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; import { pinScorers } from '../scorers/cache.js'; import { isLeaseRef } from '../scorers/entry.js'; +import { deliverPending, validateReportedPr } from './delivery.js'; import type { DriverIdentity, OutputSchema, @@ -64,7 +65,7 @@ export interface ReportInput { tokens?: TokenUsage; failure?: WorkflowFailure; /** The branch this attempt committed, when it ran in a workspace. */ - changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; + changes?: StepChanges; /** The branch and commit this attempt read, when it ran read-only in a workspace. */ read?: StepRead; agent?: ReportedAgent; @@ -264,9 +265,14 @@ function leaseData( function rememberCopy(run: RunState, plan: WorkspacePlan, driver: DriverIdentity): void { const branch = plan.contract?.branch; if (!branch) return; - const copy = (run.copies ??= {})[branch]; - if (copy) copy.owner ??= driver; - else run.copies[branch] = { owner: driver }; + const origin = plan.contract?.start.branch ?? plan.contract?.merge[0]?.branch; + const copy = run.copies?.[branch]; + if (copy) { + copy.owner ??= driver; + if (!copy.origin && !copy.commit && origin && origin !== branch) copy.origin = origin; + return; + } + (run.copies ??= {})[branch] = { owner: driver, ...(origin ? { origin } : {}) }; } /** @@ -510,6 +516,7 @@ export function stopWhere(run: RunState, under: (id: string) => boolean, failure skip(state, settledAt); break; case 'running': + case 'delivering': cancel(state, failure, settledAt); break; case 'leased': @@ -534,14 +541,41 @@ export function stopWhere(run: RunState, under: (id: string) => boolean, failure * A step takes how what it ran settled: a workflow step, the outcome, the end, and the outputs of its workflow; a * `for_each` step, the outcome and the results of its items. */ -function finishStep(state: StepState, result: WorkflowResult): void { +function deliveryDeadline(run: RunState, state: StepState): number | undefined { + const limits: number[] = []; + const root = run.definition as WorkflowDefinition & { timeout?: number }; + if (root.timeout) limits.push(Date.parse(run.createdAt) + root.timeout); + if (state.status === 'leased' && 'deadline' in state.lease && typeof state.lease.deadline === 'number') limits.push(state.lease.deadline); + for (const parent of run.steps) { + if (parent.id !== state.id && !state.id.startsWith(`${parent.id}.`)) continue; + const found = locateStep(run, parent.id); + const scope = found && found.step.workflow !== undefined ? childScope(run, found.scope, statesIn(run, found.scope), parent, found.step) : undefined; + const timeout = (found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout + ?? (scope?.definition as { timeout?: number } | undefined)?.timeout; + if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); + } + return limits.length ? Math.min(...limits) : undefined; +} + +function finishStep(run: RunState, state: StepState, definition: WorkflowStep, result: WorkflowResult): void { const settlement = { settledAt: result.settledAt, ...(result.end !== undefined ? { end: result.end } : {}), ...(result.outputs !== undefined ? { outputs: result.outputs } : {}), ...(result.results ? { results: result.results } : {}), }; - if (result.outcome === 'succeeded') succeed(state, settlement); + if (result.outcome === 'succeeded' && definition.deliver === 'pr' && (!definition.forEach || isItem(state))) { + // A workflow's children may continue one enclosing copy. Deliver that branch once at its workflow boundary. + const own = branchName(run.id, state.id, state.pass ?? 1); + const branches = [...new Set(run.steps.flatMap((child) => child.id.startsWith(`${state.id}.`) && 'changes' in child && child.changes ? [child.changes.branch] : []))]; + const branch = run.copies?.[own] ? own : branches.length === 1 ? branches[0] : undefined; + const copy = branch && run.copies?.[branch]; + if (!branch || !copy || !copy.commit || !run.base) { + fail(state, { ...settlement, failure: { code: 'delivery_failed', message: 'PR delivery needs one writable branch; set workspace: new on the workflow step.' } }); + } else { + deliver(state, { ...settlement, changes: { branch, base: run.base.commit, from: run.base.commit, to: copy.commit, commits: [] } }, deliveryDeadline(run, state)); + } + } else if (result.outcome === 'succeeded') succeed(state, settlement); else fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); } @@ -567,7 +601,7 @@ function advanceInstance( if (state.status === 'running') { const result = settleRunning(run, scope, states, state, definition); if (!('outcome' in result)) return result; - finishStep(state, result); + finishStep(run, state, definition, result); return undefined; } if (definition.forEach && !isItem(state)) { @@ -741,7 +775,7 @@ function settleScope(run: RunState, scope: Scope): Settled { return decided === 'ended' ? result : { ...result, outcome: 'failed', failure: decided }; } if (ready.length > 0) return { ready }; - if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running')) return { inProgress: true }; + if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running' || item.status === 'delivering')) return { inProgress: true }; return { unreadable: `Run ${run.id} has pending steps whose needs cannot be met` }; } } @@ -859,6 +893,12 @@ export interface NextOptions { } export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { + const envelope = await leaseNextCore(config, runId, driver, options); + if ('error' in envelope || !await deliverPending(config, runId)) return envelope; + return leaseNext(config, runId, driver, options); +} + +async function leaseNextCore(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions): Promise { driver = { kind: driver.kind, machine: driver.machine, workspace: driver.workspace }; if (options.claim) { const claimed = await claimRun(config, runId, { checkout: driver.workspace }); @@ -888,7 +928,9 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv return answer(doneData(run)); }; const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => ( - answer({ wait: { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) + answer({ wait: run.steps.some((step) => step.status === 'delivering') + ? { reason: 'PR delivery is in progress; poll next', delivery: true } + : { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) ); if (run.status !== 'running') { // Settled by an earlier next, which queued the end record unless usage @@ -1099,6 +1141,10 @@ export function parseReport(value: unknown): ParsedReport | string { return 'changes.commits must be an array of strings'; } report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; + if (value.pr_url !== undefined) { + // Invalid types are dropped with a value-free reason by reportAttempt too. + report.changes.pr_url = value.pr_url as string; + } } if (record.read !== undefined) { const read = ReadSchema.safeParse(record.read); @@ -1115,7 +1161,12 @@ export function parseReport(value: unknown): ParsedReport | string { if (typeof used === 'string') return used; if (used) report.used = used; } - return { report, ignored: unreadFields(record, REPORT_FIELDS) }; + const ignored = unreadFields(record, REPORT_FIELDS); + if (isRecord(record.changes)) { + const known = new Set(['branch', 'base', 'from', 'to', 'commits', 'pr_url']); + ignored.push(...Object.keys(record.changes).filter((key) => !known.has(key)).map((key) => `changes.${key}`)); + } + return { report, ignored }; } /** A `next` payload: who the driver is, and what else it tells `next`. */ @@ -1274,7 +1325,20 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: return error('judge_lease', `Attempt ${report.attempt} is a judge lease, which workflow report does not settle. Submit its answer with: teamai score --lease ${report.attempt} --submit --stdin`); } const install = await installOwner(config); - return withRun(config, runId, async (run) => { + const dropped: Array<{ field: string; reason: string }> = []; + if (report.changes && report.changes.pr_url !== undefined) { + report = structuredClone(report); + const run = await readRun(config, runId); + if (typeof run !== 'string') { + const result = await validateReportedPr(run, report.changes!); + if (result.url) report.changes!.pr_url = result.url; + else { + delete report.changes!.pr_url; + dropped.push({ field: 'changes.pr_url', reason: result.reason! }); + } + } + } + const envelope = await withRun(config, runId, async (run) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { return { save: false, envelope: error('repeated_report', `Attempt ${report.attempt} was already reported`) }; @@ -1335,7 +1399,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: model: report.model, session: report.session, tokens: report.tokens, - ...(report.changes ? { changes: report.changes } : {}), + ...(report.changes ? { changes: { ...report.changes, prRecordedAt: report.changes.pr_url ? run.revision : undefined } } : {}), ...(report.read ? { read: report.read } : {}), agent: settleObservation(report, defined?.agent), ...(report.used ? { used: report.used } : {}), @@ -1348,11 +1412,12 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: copy.commit = report.changes.to; run.copies[report.changes.branch] = copy; } + const successful = { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }; const settledStep = outcome === 'released' ? release(step) - : outcome === 'success' ? succeed(step, { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }) + : outcome === 'success' ? defined?.deliver === 'pr' ? deliver(step, successful, deliveryDeadline(run, step)) : succeed(step, successful) : fail(step, { ...settlement, failure }); // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. - const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch); + const cleanup = outcome === 'success' && settledStep.status !== 'delivering' && contract !== undefined && !branchStillUsed(run, contract.branch); return { save: true, envelope: ok({ @@ -1363,4 +1428,22 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: }), }; }); + if ('error' in envelope) return envelope; + const pending = await readRun(config, runId); + if (typeof pending !== 'string' && pending.status === 'running' && pending.steps.some((step) => step.status === 'delivering')) { + // Reconcile before provider I/O too: limits reached by this report must stop delivery before it opens a PR. + const checked = await leaseNextCore(config, runId, report.driver, { canTakeLease: false, + running: pending.steps.flatMap((step) => step.status === 'leased' ? [step.attempt] : []), + }); + if (!('error' in checked)) await deliverPending(config, runId); + const run = await readRun(config, runId); + const step = typeof run !== 'string' && run.steps.find((state) => 'attempt' in state && state.attempt === report.attempt); + if (step) { + envelope.data.revision = run.revision; + envelope.data.step = { id: step.id, status: step.status, outcome: step.status === 'failed' || step.status === 'cancelled' ? 'failed' : 'success' }; + if (step.status === 'succeeded' && report.changes && !branchStillUsed(run, report.changes.branch)) envelope.data.cleanup = true; + } + } + if (dropped.length) envelope.data.dropped = dropped; + return envelope; } diff --git a/src/workflow/fan-out.ts b/src/workflow/fan-out.ts index 459339c29..cd1662fcb 100644 --- a/src/workflow/fan-out.ts +++ b/src/workflow/fan-out.ts @@ -95,7 +95,7 @@ export function afterStanding(after: StepState[]): 'ready' | 'waiting' | 'skip' /** How many items are started and not yet settled, which `max_parallel` bounds. */ export function startedItems(items: StepState[]): number { - return items.filter((item) => item.status === 'leased' || item.status === 'running').length; + return items.filter((item) => item.status === 'leased' || item.status === 'running' || item.status === 'delivering').length; } /** diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index 1329499f7..9f87e51b0 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -1097,6 +1097,10 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options if ('lease' in next.data && room()) continue; if (inFlight.size === 0 && dying.size === 0 && !prompt && !question) { // A wait for a call that could not take a lease says nothing about what is left; ask again with room. + if ((next.data.wait as { delivery?: boolean } | undefined)?.delivery) { + await new Promise((resolve) => setTimeout(resolve, 100)); + continue; + } if (!canTakeLease) continue; const reason = typeof (next.data.wait as { reason?: unknown }).reason === 'string' ? (next.data.wait as { reason: string }).reason diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 65595a178..0015a9c2d 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -20,7 +20,7 @@ import { modelValueProblem } from '../models/aliases.js'; const HEADER_KEYS = new Set(['schemaVersion', 'name', 'description', 'inputs', 'defaults', 'ends', 'outputs', 'loop', 'steps']); const ACTION_KEYS = Object.keys(STEP_ACTIONS); -const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session', 'scorers']); +const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session', 'scorers', 'deliver']); const STEP_LOOP_KEYS = new Set(['back_to', 'while', 'max_iterations']); const FOR_EACH_KEYS = new Set(['items', 'max_parallel']); const WORKFLOW_LOOP_KEYS = new Set(['while', 'max_iterations']); @@ -497,6 +497,14 @@ function parseBody( }); } const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); + const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); + const deliver = scalarString(deliverPair?.value)?.trim(); + if (deliverPair && (deliver !== 'pr' || workspace === 'read-only')) { + diagnostics.push({ + ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, 'deliver must be pr on a writable branch'), + code: 'bad_delivery', + }); + } let session: WorkflowStep['session']; if (sessionPair) { const sessionText = scalarString(sessionPair.value)?.trim(); @@ -544,6 +552,7 @@ function parseBody( ...(breakSource !== undefined ? { break: breakSource } : {}), ...(forEach ? { forEach } : {}), ...(workspace ? { workspace } : {}), + ...(deliver === 'pr' ? { deliver } : {}), ...(session ? { session } : {}), }); } diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts new file mode 100644 index 000000000..02ee366ae --- /dev/null +++ b/src/workflow/pr-url.ts @@ -0,0 +1,16 @@ +/** Public run records accept HTTPS review links on any host, but never URL credentials or session parameters. */ +export function sanitizePrUrl(value: unknown): { url: string; reason?: never } | { url?: never; reason: string } { + if (typeof value !== 'string') return { reason: 'must be a URL string' }; + if (value.length > 512) return { reason: 'exceeds 512 characters' }; + try { + const url = new URL(value); + if (url.protocol !== 'https:') return { reason: 'must use HTTPS' }; + if (url.username || url.password) return { reason: 'contains credentials' }; + url.search = ''; + url.hash = ''; + const sanitized = url.toString(); + return sanitized.length <= 512 ? { url: sanitized } : { reason: 'exceeds 512 characters' }; + } catch { + return { reason: 'is not a valid URL' }; + } +} diff --git a/src/workflow/records.ts b/src/workflow/records.ts index a29fe9130..8e07c517b 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -27,6 +27,7 @@ import { installOwner, runLockPath } from './store.js'; import { settled, VISIBLE_OUTCOMES, type StepPass, type StepState, type VisibleOutcome } from './step-state.js'; import type { TokenUsage } from '../tool-launch/stdout.js'; import type { DriverIdentity, RunState, RunStatus, WorkflowFailure } from './types.js'; +import { sanitizePrUrl } from './pr-url.js'; /** * The failure codes the engine writes, then those the local driver sets. A code a model writes into its result, or @@ -37,6 +38,7 @@ const RECORDED_FAILURES = [ 'loop_exhausted', 'invalid_output', 'invalid_template', 'invalid_inputs', 'invalid_items', 'item_cycle', 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', + 'delivery_failed', 'timeout', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; @@ -96,6 +98,8 @@ export type WorkflowRunRecord = { stepCounts?: { total: number; succeeded: number; failed: number; skipped: number; cancelled: number }; /** Settled attempts across the run — the cumulative iteration count. */ iterations?: number; + /** Validated review links, first appearance order. Public on a public team repo; omitted before this version. */ + prs?: string[]; username: string; publishedAt: string; }; @@ -209,6 +213,13 @@ export function buildEndRecord( ]), ...(run.archive ?? []).map((archived) => entry(archived.step, archived.pass, archived)), ]; + const changes = [ + ...run.steps.flatMap((step) => [...(step.passes ?? []).flatMap((pass) => pass.changes ?? []), ...('changes' in step && step.changes ? [step.changes] : [])]), + ...(run.archive ?? []).flatMap((pass) => pass.changes ?? []), + ]; + const prs = [...new Set(changes.filter((item) => item.prRecordedAt !== undefined) + .sort((a, b) => a.prRecordedAt! - b.prRecordedAt!) + .flatMap((item) => sanitizePrUrl(item.pr_url).url ?? []))]; return { ...buildStartRecord(run, driver, username, now), phase: 'end', @@ -230,6 +241,7 @@ export function buildEndRecord( // Cumulative when present; runs from before the counter fall back to the // bounded history length (their true count is unknowable now). iterations: run.settledCount ?? run.settledAttempts.length, + ...(prs.length ? { prs } : {}), }; } diff --git a/src/workflow/run-controls.ts b/src/workflow/run-controls.ts index f758ada71..b1bcb84f1 100644 --- a/src/workflow/run-controls.ts +++ b/src/workflow/run-controls.ts @@ -9,7 +9,7 @@ import { publishRunEnd } from './handoff.js'; import { archiveUnder, isUnder } from './loop.js'; import { publishRunRecords, removeQueuedEndRecord } from './records.js'; import { contextOf, localId, statesIn } from './scope.js'; -import { answerHitl, cancel, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; +import { answerHitl, cancel, deliver, repend, resetIteration, resetPending, settled, start, stopped, type HitlAnswer, type StepState } from './step-state.js'; import type { RunState, WorkflowEnvelope } from './types.js'; async function controlRun( @@ -119,7 +119,7 @@ export async function cancelRun(config: LocalConfig, runId: string, step?: strin } else { const state = run.steps.find((state) => state.id === step); if (!state) return missingStepError(run, step); - if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased or running steps can be cancelled`); + if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running' && state.status !== 'delivering') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased, running or delivering steps can be cancelled`); const located = locateStep(run, step); if (!located) return missingStepError(run, step); changed = state.status === 'running' ? stopWhere(run, isUnder(state.id), failure, run.revision) : []; @@ -228,6 +228,7 @@ function retryInstance(run: RunState, state: StepState): void { cancel(state, failure, run.revision); break; case 'running': + case 'delivering': stopWhere(run, isUnder(state.id), failure, run.revision); break; case 'succeeded': @@ -268,7 +269,10 @@ export async function retryStep(config: LocalConfig, runId: string, step: string const reopened = run.status !== 'running'; for (const container of containers) { if (container.child === target) { - retryInstance(run, target); + if (target.status === 'failed' && target.failure?.code === 'delivery_failed' && target.changes) { + const { status: _status, outcome: _outcome, failure: _failure, message: _message, ...settlement } = target; + deliver(target, { ...settlement, settledAt: run.revision + 1 }); + } else retryInstance(run, target); resetIteration(target); } for (const state of container.states) { diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 4410f165a..d360a834b 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -31,6 +31,8 @@ const ChangesSchema = z.object({ from: z.string(), to: z.string(), commits: z.array(z.string()), + pr_url: z.string().optional(), + prRecordedAt: z.number().int().optional(), }); /** What a read-only step inside a copy reports: the branch it ran on and the commit it read. It wrote nothing. */ @@ -137,6 +139,8 @@ const settledFields = { results: ResultsSchema.optional(), settledAt: z.number().int().default(0), attempt: z.string().optional(), + /** The original execution bound continues while its completed work is delivered. */ + deliveryDeadline: z.number().optional(), ...report, }; @@ -154,13 +158,14 @@ const SucceededSchema = StepBaseSchema.extend({ }); const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional() }); const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); -const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema }); +const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }); export const StepStateSchema = z.discriminatedUnion('status', [ PendingSchema, WaitingSchema, StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }), RunningSchema, + StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields }), SucceededSchema, FailedSchema, SkippedSchema, @@ -287,13 +292,22 @@ export function succeed(state: StepState, settlement: Settlement) return become(state, { status: 'succeeded', outcome: 'success', ...settlement }); } +/** Agent work is durable, but dependents wait for delivery to finish. */ +export function deliver(state: StepState, settlement: Settlement, deadline?: number): StepIn<'delivering'> { + return become>(state, { status: 'delivering', ...settlement, ...(deadline !== undefined ? { deliveryDeadline: deadline } : {}) }); +} + export function fail(state: StepState, settlement: Settlement): FailedStep { return become(state, { status: 'failed', outcome: 'failed', ...settlement }); } /** An open step is cancelled before it settles. A leased one keeps its invalidated attempt. */ -export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running'>, failure: WorkflowFailure, settledAt: number): CancelledStep { - return become(state, { status: 'cancelled', settledAt, ...(state.status === 'leased' ? { attempt: state.attempt } : {}), failure }); +export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { + // Delivery follows completed agent work. Cancellation must retain that work's usage and any review link. + const reported = state.status === 'delivering' + ? Object.fromEntries(SETTLED_KEYS.filter((key) => state[key] !== undefined).map((key) => [key, state[key]])) + : state.status === 'leased' ? { attempt: state.attempt } : {}; + return become(state, { ...reported, status: 'cancelled', settledAt, failure }); } /** The current pass of a settled step, as `passes` keeps it once a loop runs the step again; undefined while it has not settled. */ @@ -351,6 +365,7 @@ export function stopped(step: StepState): boolean { case 'waiting': case 'leased': case 'running': + case 'delivering': case 'succeeded': case 'skipped': return false; @@ -374,6 +389,7 @@ export function settled(step: StepState | undefined): step is Extract; + copies?: Record; /** * Sessions a step resumed and continued under another id, in report order: a copy on another machine or in another * tool. `to` holds the turns of `from` and the ones that step added, so a step that would resume `from` resumes `to`. diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 19e32f012..4b1b32160 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -17,6 +17,9 @@ export interface StepChanges { from: string; to: string; commits: string[]; + pr_url?: string; + /** Engine revision at which the validated URL first appeared. Never accepted from a driver. */ + prRecordedAt?: number; } /** @@ -128,13 +131,26 @@ function workflowChildren(run: RunState, step: StepState): StepState[] { /** Branches a step merges, `after` items first, then its other needs. An omitted `needs` uses the parent step's needs. */ function mergeChanges(run: RunState, scope: Scope, state: StepState, definition: WorkflowStep): StepChanges[] { const ordered: Array<{ changes: StepChanges; settledAt: number }> = []; + const visited = new Set(); const push = (step: StepState | undefined): void => { - if (!step) return; + if (!step || visited.has(step.id)) return; + visited.add(step.id); const changes = changesOf(step); // A for_each step keeps each item's branch on that item, and a workflow step keeps each child's on that child. if (!changes) { for (const item of itemsOf(run, step)) push(item); for (const child of workflowChildren(run, step)) push(child); + // A skipped or read-only need wrote no branch, but its earlier dependencies still supply the copy's origin. + let id: string | undefined = step.id; + while (id) { + const defined = stepDefinitions(run)(id); + const prefix: string = id.includes('.') ? id.slice(0, id.lastIndexOf('.') + 1) : ''; + if (defined?.needs !== undefined) { + for (const need of defined.needs) push(run.steps.find((item) => item.id === `${prefix}${need}`)); + break; + } + id = prefix ? prefix.slice(0, -1) : undefined; + } return; } const settledAt = 'settledAt' in step ? step.settledAt : 0; From 16f84d06f2877e5c0c39133097714c695df8b60d Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:25:50 +0800 Subject: [PATCH 60/77] fix(workflow): retain every enclosing delivery timeout - Carry both a workflow step timeout and its nested workflow timeout into the delivery deadline, keeping the earliest bound rather than preferring one. - Keep the root and lease deadlines in the same minimum so delivery cannot relax the constraints established by the requirements implementation. - Verify all 29 delivery regressions, typecheck, lint and the build-first real CLI PR creation/reuse, generic-host recovery and required-approval cases. --- src/workflow/engine.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 7f3a8de8a..bbc49622c 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -550,9 +550,9 @@ function deliveryDeadline(run: RunState, state: StepState): number | undefined { if (parent.id !== state.id && !state.id.startsWith(`${parent.id}.`)) continue; const found = locateStep(run, parent.id); const scope = found && found.step.workflow !== undefined ? childScope(run, found.scope, statesIn(run, found.scope), parent, found.step) : undefined; - const timeout = (found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout - ?? (scope?.definition as { timeout?: number } | undefined)?.timeout; - if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); + const bounds = [(found?.step as (WorkflowStep & { timeout?: number }) | undefined)?.timeout, + (scope?.definition as { timeout?: number } | undefined)?.timeout]; + for (const timeout of bounds) if (timeout && 'startedAt' in parent && typeof parent.startedAt === 'number') limits.push(parent.startedAt + timeout); } return limits.length ? Math.min(...limits) : undefined; } From e6274b84494e7ad328139c7958a0b7618504cd93 Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Fri, 23 Oct 2026 16:31:56 +0800 Subject: [PATCH 61/77] fix(workflow): recheck cancellation before opening review requests - Reload the durable run and verify the same delivering pass/attempt after an open-request lookup returns no match, before starting any PR creation request. Cancellation during lookup now prevents creation; already in-flight creation still retains its safe receipt without reviving cancelled work. - Reject deliver: pr on commands with an implicitly read-only workspace, while retaining prompts that inherit a writable enclosing workflow. Clarify the command requirement in both language guides. - Add a regression that cancels during lookup and previously observed an unwanted create call, plus parser coverage for implicit read-only commands and valid inherited prompt workspaces. - Verify 118 delivery/parser/provider/guide tests, typecheck, lint and a build-first real CLI delivery, reuse, recovery and approval round trip. --- docs/guide/workflow.md | 2 ++ docs/guide/zh-CN/workflow.md | 2 ++ src/__tests__/workflow-delivery.test.ts | 15 +++++++++++++++ src/workflow/delivery.ts | 12 +++++++++--- src/workflow/parse.ts | 3 ++- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c05b7d796..a1aa008e4 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -253,6 +253,8 @@ steps: The PR targets the nearest delivered branch along the branch's recorded origin; `for_each.after` comes before other `needs`, and the first child inherits the enclosing workflow's dependencies. Skipped or read-only dependencies are passed over. If no earlier branch delivered, the target is the run's original branch. Start from a named branch: a detached checkout has no PR target. An open PR for the same head and target is reused. +Commands that deliver must explicitly choose `workspace: new` or `continue`: omitting workspace makes a command read-only even inside a writable workflow. Prompts can inherit an enclosing writable workspace. + Agent completion is followed by a durable `delivering` state. Dependents wait for it. Git and provider operations happen outside the run lock. A PR creation failure reports `delivery_failed` and names the preserved branch. Fix provider authentication, run `teamai workflow retry `, then `teamai workflow resume `: this retries delivery on that branch without rerunning the agent. Push failures remain `push_failed` and retain the worktree. A completed composite may already have removed its worktree; its committed, pushed branch remains. Generic Git hosts support transport only: create a PR yourself and report its URL, or use a provider that supports PR creation. No interactive login starts during delivery. Before opening a PR, TeamAI checks the delivered commit against the run's base commit. Changes to CI, hooks, agent instructions, rules, MCP and equivalent tool configuration are rejected, with the paths named. The shared agent configuration paths are the same ones restored before workspace execution. This check includes changes inherited from earlier branches and deletions/renames. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 9feea5fdc..ca7562348 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -253,6 +253,8 @@ steps: PR 的目标沿分支记录的来源，追溯到最近已经交付的分支。`for_each.after` 优先于其他 `needs`，第一个子步骤继承外层工作流的依赖；跳过或只读的依赖会继续向前追溯。之前没有交付分支时，目标为运行开始时的分支。因此请从有名称的分支开始；detached checkout 没有 PR 目标。同一源分支和目标分支已有打开的 PR 时会复用。 +交付 PR 的命令必须显式选择 `workspace: new` 或 `continue`；即使位于可写工作流中，省略 workspace 的命令仍为只读。提示词可以继承外层的可写工作区。 + agent 完成后，步骤进入持久的 `delivering` 状态，后续依赖等待交付结束。Git 和 provider 操作不占用运行锁。创建 PR 失败时以 `delivery_failed` 结束，消息写明保留的分支。修好认证后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，只重试这条分支的交付，不重跑 agent。推送失败仍为 `push_failed`，并保留 worktree。组合工作流结束时可能已移除 worktree，但提交和远程分支仍在。通用 Git 主机只支持传输：请自行创建 PR 并上报 URL，或使用支持创建 PR 的 provider。交付不会启动交互登录。 创建 PR 前，TeamAI 将交付提交与运行的基准提交比较。CI、hooks、agent 指令、rules、MCP 和同类工具配置的修改会被拒绝，并列出路径。共享 agent 配置路径与工作区执行前恢复的路径相同。检查也覆盖从前序分支继承的修改，以及删除和重命名。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 7ab3666af..c1167c8d1 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -77,6 +77,10 @@ describe('workflow PR delivery contract', () => { const parsed = parseWorkflow('ship.yaml', yaml); expect(parsed.ok && parsed.definition.steps[0]).toMatchObject({ deliver: 'pr' }); expect(parseWorkflow('ship.yaml', yaml.replace('deliver: pr', 'deliver: merge')).ok).toBe(false); + expect(parseWorkflow('ship.yaml', yaml.replace(' workspace: new\n', '')).ok).toBe(false); + const inherited = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n prompt: Edit the code.\n deliver: pr\n'; + expect(parseWorkflow('ship.yaml', inherited).ok).toBe(true); + expect(parseWorkflow('ship.yaml', inherited.replace('prompt: Edit the code.', 'command: echo edit')).ok).toBe(false); }); it('keeps the URL for validation and names unread nested fields without values', () => { @@ -269,6 +273,17 @@ describe('durable PR delivery', () => { expect(run.steps[0]).toMatchObject({ status: 'cancelled', changes: { pr_url: 'https://github.com/fixture/app/pull/99' } }); expect(buildEndRecord(run, driver, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/99']); }); + it('does not create a PR when cancellation completed during the open-request lookup', async () => { + const api = provider(); + const { config, cwd, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); + api.lookup.mockImplementation(async () => { + await cancelRun(config, runId, 'write'); + return undefined; + }); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect(api.create).not.toHaveBeenCalled(); + expect((await state(config, runId)).steps[0]).toMatchObject({ status: 'cancelled' }); + }); it('keeps an execution deadline across delivery and makes no provider call once it expired', async () => { const api = provider(); diff --git a/src/workflow/delivery.ts b/src/workflow/delivery.ts index 2884f7070..b24d9ecbc 100644 --- a/src/workflow/delivery.ts +++ b/src/workflow/delivery.ts @@ -85,7 +85,7 @@ export async function validateReportedPr(run: RunState, changes: StepChanges): P } } -async function createDelivery(run: RunState, state: Extract): Promise { +async function createDelivery(config: LocalConfig, run: RunState, state: Extract): Promise { const remaining = (): number => { const ms = state.deliveryDeadline === undefined ? 30_000 : Math.min(30_000, state.deliveryDeadline - Date.now()); if (ms <= 0) throw new Error('The execution deadline expired before delivery completed.'); @@ -114,9 +114,15 @@ async function createDelivery(run: RunState, state: Extract step.id === state.id); + if (typeof latest === 'string' || latest.status !== 'running' || !active || active.status !== 'delivering' + || active.pass !== state.pass || active.attempt !== state.attempt || active.settledAt !== state.settledAt) { + throw new Error('Delivery was cancelled or replaced during PR lookup; no PR was created.'); + } + request.spawnTimeoutMs = remaining(); try { url = await provider.createPullRequest(request); } catch (error) { // A provider's response body or CLI output may contain credentials. Keep its established transport-only @@ -146,7 +152,7 @@ export async function deliverPending(config: LocalConfig, runId: string): Promis if (!state) break; let url: string | undefined; let failure: string | undefined; - try { url = await createDelivery(run, state); } + try { url = await createDelivery(config, run, state); } catch (error) { failure = error instanceof Error ? error.message : 'PR delivery failed'; } const applied = await withRun(config, runId, async (current) => { const step = current.steps.find((item) => item.id === state.id); diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 0015a9c2d..6b60a7e01 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -499,7 +499,8 @@ function parseBody( const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); const deliver = scalarString(deliverPair?.value)?.trim(); - if (deliverPair && (deliver !== 'pr' || workspace === 'read-only')) { + // A command without workspace is always read-only, including inside a writable workflow. + if (deliverPair && (deliver !== 'pr' || workspace === 'read-only' || (commandPair && workspace === undefined))) { diagnostics.push({ ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, 'deliver must be pr on a writable branch'), code: 'bad_delivery', From 3e94b50be255cdfbc40fc5fa8f01c9004c7ac6c8 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 19:28:37 +0200 Subject: [PATCH 62/77] fix(workflow): read git's own exit code, so an error is not "not an ancestor" git() read error.status, which execFile never sets, so every failure came back as exit code 1. merge-base --is-ancestor answers 1 for "not an ancestor" and 128 for an error, so delivery's ancestry check could turn an unreadable commit into a rewritten branch. git() now reads error.code, and the ancestry check treats any other code as unreadable. Red: 'tells a commit that is not an ancestor from a git error' failed with expected 1 to be 128. --- src/__tests__/workflow-delivery.test.ts | 14 ++++++++++++++ src/workflow/pr-delivery.ts | 3 ++- src/workflow/worktree.ts | 3 +-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 2b6c49a84..0599397f8 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -727,6 +727,20 @@ describe('protected paths', () => { }); }); +describe('reading the delivered branch', () => { + it('tells a commit that is not an ancestor from a git error', async () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-git-code-')); + execFileSync('git', ['init', '-q', repo]); + const missing = '0'.repeat(40); + + try { + expect((await worktree.git(repo, ['merge-base', '--is-ancestor', missing, missing], { timeoutMs: 5_000 })).code).toBe(128); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); +}); + describe('reported URLs from a driver', () => { it('drops a URL with credentials without failing the report, and never stores, echoes or publishes it', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 81a23a252..ea74875ee 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -211,7 +211,8 @@ async function contains({ runId, checkout }: DeliveryJob, { commit, ancestor }: if (fetched.missing.includes(commit)) return { fault: { kind: 'unreadable' } }; // A commit the checkout holds brings its history, so an ancestor still missing is not in it. if (fetched.missing.length > 0) return { contains: false }; - return { contains: (await git(checkout, ['merge-base', '--is-ancestor', ancestor, commit])).code === 0 }; + const { code } = await git(checkout, ['merge-base', '--is-ancestor', ancestor, commit]); + return code === 0 || code === 1 ? { contains: code === 0 } : { fault: { kind: 'unreadable' } }; } const LISTED_FILES = 20; diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 0e5b04e82..07a904dbe 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -64,8 +64,7 @@ export function git(cwd: string, args: string[], options?: { input: string } | { if (options === undefined || !('input' in options)) { return new Promise((resolve) => { execFile('git', args, { cwd, env: { ...process.env, ...options?.env }, windowsHide: true, timeout: options?.timeoutMs }, (error, stdout, stderr) => { - const failed = error as NodeJS.ErrnoException & { status?: number } | null; - const code = typeof failed?.status === 'number' ? failed.status : failed ? 1 : 0; + const code = typeof error?.code === 'number' ? error.code : error ? 1 : 0; resolve({ code, stdout: String(stdout), stderr: String(stderr) }); }); }); From d609e160a53a1900508feaea3790137e3ae3a50b Mon Sep 17 00:00:00 2001 From: Nathan Shan Date: Sun, 25 Oct 2026 01:36:55 +0800 Subject: [PATCH 63/77] =?UTF-8?q?fix(workflow):=20=E5=9C=A8=E8=B7=A8?= =?UTF-8?q?=E6=9C=BA=E5=99=A8=E4=BA=A4=E6=8E=A5=E6=97=B6=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E5=BE=85=E4=BA=A4=E4=BB=98=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 保留主分支发布与恢复、会话续接和评分状态，合并命名空间及交付诊断码，不恢复已移除的无锁写入接口。 - 为 delivering 步骤、交付截止时间和来源分支补齐快照规则与交接 schema，防止发布后丢失待交付工作。 - 使用当前 updateRun 接口构建测试状态；新增快照交接回归，相关最终聚焦测试全部通过。 - 真实 CLI PR 创建与复用、通用 Git 保留及审批保护用例通过；构建、类型检查和 lint 通过。 --- docs/dev/workflow-engine.md | 2 ++ src/__tests__/workflow-delivery.test.ts | 23 ++++++++++++++++++++++- src/__tests__/workflow-records.test.ts | 2 +- src/workflow/engine.ts | 1 + src/workflow/snapshot.ts | 6 +++--- src/workflow/step-state.ts | 2 ++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 98eccd014..213ca0898 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -43,3 +43,5 @@ Each rule below spans more than one file. The rule names the code that owns it, ## Tests Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass the protocol scenarios in `src/workflow/conformance.ts`, and a driver that takes workspace leases the git scenarios there too. If a driver can see a new behavior, add a scenario there. + +Pending PR delivery is part of the handoff snapshot: preserve the `delivering` step, its deadline and the copy's origin branch. Claiming releases execution leases but does not rerun completed agent work while delivery is pending. `snapshot.ts` and `StepCheckpointSchema` must both accept those fields. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index c1167c8d1..051b41abd 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -6,10 +6,12 @@ import path from 'node:path'; import { execFileSync } from 'node:child_process'; import { parseWorkflow } from '../workflow/parse.js'; import { leaseNext, parseReport, reportAttempt, startRun } from '../workflow/engine.js'; +import { projectRun } from '../workflow/snapshot.js'; +import { takeOver } from '../workflow/step-state.js'; import { sanitizePrUrl } from '../workflow/pr-url.js'; import { deliveryTarget, protectedDeliveryPaths, validateReportedPr } from '../workflow/delivery.js'; import { syncWorkflowDefinitions } from '../workflow/cache.js'; -import { readRun, writeRun } from '../workflow/store.js'; +import { readRun, updateRun } from '../workflow/store.js'; import { runLocalDriver } from '../workflow/local-driver.js'; import { GitHubProvider } from '../providers/github/index.js'; import { buildEndRecord } from '../workflow/records.js'; @@ -18,6 +20,11 @@ import { settled, stopped } from '../workflow/step-state.js'; import type { LocalConfig } from '../types.js'; import type { RunState } from '../workflow/types.js'; +async function writeRun(config: LocalConfig, run: RunState): Promise { + const result = await updateRun(config, run.id, async () => ({ value: undefined, write: run })); + expect(result.status).toBe('done'); +} + const dirs: string[] = []; afterEach(() => { vi.restoreAllMocks(); @@ -207,6 +214,20 @@ describe('durable PR delivery', () => { expect(JSON.stringify(run)).not.toContain('session='); }); + it('publishes and hands over a pending delivery without dropping its origin or deadline', async () => { + const { config, runId } = await fixture(` - id: write\n command: echo test\n`); + const run = await state(config, runId); + const branch = `teamai/${runId}/write/p1`; + const commit = run.base!.commit; + run.copies = { [branch]: { owner: driver, commit, origin: 'main' } }; + run.steps = [{ id: 'write', status: 'delivering', settledAt: 1, deliveryDeadline: 12345, + changes: { branch, base: commit, from: commit, to: commit, commits: [] } }]; + const projected = projectRun(run, {}); + expect(projected, JSON.stringify(projected)).toMatchObject({ ok: true, value: { copies: { [branch]: { origin: 'main' } }, steps: [{ status: 'delivering', deliveryDeadline: 12345 }] } }); + if (!projected.ok) throw new Error(projected.error.message); + expect(takeOver(projected.value.steps[0]!, 'new')).toMatchObject({ status: 'delivering', deliveryDeadline: 12345 }); + }); + it('can cancel a persisted delivery without treating it as settled success', async () => { const { config, runId } = await fixture(` - id: write\n command: ${command}\n workspace: new\n deliver: pr\n`); const run = await state(config, runId); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index 8674c4c4c..2231c56b8 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -205,7 +205,7 @@ describe('#1083 workflow run records', () => { it('publishes the failure code of every step failure teamai writes, and only codes it can name', () => { // Codes a module writes, as `code: '…'`, `error('…', …)` or `failure('…', …)`, that never become a run's failure: every module of src/workflow and src/scorers, and every other one that imports src/scorers or answers a WorkflowEnvelope. const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'workflow_namespace_conflict', 'prompt_namespace_conflict', 'bad_delivery'] satisfies Array; - const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'invalid_report', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'dropped', 'busy', 'workflow_recursion', 'workflow_not_found', + const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'invalid_report', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'busy', 'workflow_recursion', 'workflow_not_found', // Refusals of a run handover, answered to the command or driver that asked for it. 'not_owner', 'unsupported_repo', 'reports_unreachable', 'reports_failed', 'run_not_found', 'run_not_published', 'run_mismatch', 'state_conflict', 'state_moved', 'definition_mismatch', 'records_pending', 'snapshot_unsupported', 'secret_detected', 'local_path_detected', 'origin_unavailable', 'commit_missing', diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index bbc49622c..85cbb0969 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -30,6 +30,7 @@ import { pinScorers } from '../scorers/cache.js'; import { isLeaseRef } from '../scorers/entry.js'; import { deliverPending, validateReportedPr } from './delivery.js'; import type { + WorkflowDefinition, DriverIdentity, OutputSchema, RunState, diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index 5199f9703..1d8e39612 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -53,7 +53,7 @@ const BodySchema = z.object({ round: z.number().int().optional(), base: z.object({ ref: z.string().optional(), commit: CommitIdSchema }).optional(), /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ - copies: z.record(z.object({ commit: CommitIdSchema.optional() })), + copies: z.record(z.object({ commit: CommitIdSchema.optional(), origin: z.string().optional() })), sessionContinuations: z.array(SessionContinuationSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ scorers: z.record(z.object({ sourceHash: z.string() })).optional(), @@ -196,7 +196,7 @@ const RUN_RULES = { const STEP_RULES = { id: 'publish', pass: 'publish', passes: 'rewrite', answer: 'rewrite', item: 'publish', inputs: 'publish', - iteration: 'publish', round: 'publish', status: 'publish', attempt: 'publish', lease: 'rewrite', outcome: 'publish', + iteration: 'publish', round: 'publish', deliveryDeadline: 'publish', status: 'publish', attempt: 'publish', lease: 'rewrite', outcome: 'publish', outputs: 'publish', end: 'publish', results: 'publish', settledAt: 'publish', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', @@ -220,7 +220,7 @@ const FAILURE_RULES = { 'code': 'publish', message: 'rewrite' } satisfies FieldR // A lease another machine held is released when the run is handed over, so the session it offered stays here. const LEASE_RULES = { attempt: 'publish', revision: 'publish', driver: 'rewrite', session: 'drop' } satisfies FieldRules; -const COPY_RULES = { owner: 'drop', commit: 'publish' } satisfies FieldRules[string]>; +const COPY_RULES = { owner: 'drop', commit: 'publish', origin: 'publish' } satisfies FieldRules[string]>; const FAILURE_REWRITES = { message: () => ({ message: HANDED_FAILURE_MESSAGE }), diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index d360a834b..b769ff141 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -186,6 +186,7 @@ export const StepCheckpointSchema = z.discriminatedUnion('status', [ status: z.literal('leased'), attempt: z.string(), lease: z.object({ attempt: z.string(), revision: z.number(), driverKind: z.string() }), }), RunningSchema, + StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields }), SucceededSchema, FailedSchema, SkippedSchema, @@ -265,6 +266,7 @@ export function takeOver(step: StepCheckpoint, mode: WorkspaceMode | undefined): case 'pending': case 'waiting': case 'running': + case 'delivering': case 'succeeded': case 'failed': case 'skipped': From e8383b4db92892214298bbd419b640385ab15359 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 20:16:30 +0200 Subject: [PATCH 64/77] fix(workflow): read origin's tip again before opening a pull request A sibling `continue` step can push, and report, a commit while delivery looks up the branch's open request. The create gate only checked that the step still delivered the same key, so the create went ahead with content no check had read. deliverPr now reads origin's tip again after the lookup, and the gate also refuses when `copies[branch].commit` is no longer the job's `recordedHead`. Either answers `busy`: nothing is listed or opened, the step stays delivering, and the next drain checks the new tip. Red: "opens nothing when a sibling ... and checks that commit on the next call" failed in all three cases with the create already made and `work.impl` delivered (`done: succeeded`, a wait only for `work.docs`, or the `work.docs` lease handed back) instead of `wait.delivering: ['work.impl']`. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 50 +++++++++++++++++++++++++ src/workflow/engine.ts | 9 +++-- src/workflow/pr-delivery.ts | 16 +++++--- 4 files changed, 67 insertions(+), 10 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 5d3c0b70f..96ff24c55 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy`, not a refusal: the step stays `delivering`, and the drain stops for that call. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 0599397f8..76d2a09f2 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1229,6 +1229,56 @@ describe('retrying and cancelling a delivery', () => { expect(create).not.toHaveBeenCalled(); }); + it.each([ + ['pushes and records a protected commit while the lookup runs', 'lookup', true], + ['pushes a protected commit while the lookup runs, and records it later', 'lookup', false], + ['pushes and records a protected commit once delivery read origin\'s branch for the last time', 'last read', true], + ] as const)('opens nothing when a sibling %s, and checks that commit on the next call', async (_case, when, recorded) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + let sibling: string | undefined; + const siblingPushes = async () => { + sibling = pushOnto(root, branch, { '.github/workflows/ci.yml': 'on: push\n' }); + if (recorded) await succeed(config, runId, docs, { to: sibling }); + }; + let looked = false; + const read = worktree.git; + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + const answer = await read(...args); + if (when === 'last read' && looked && sibling === undefined && args[1].includes('ls-remote')) await siblingPushes(); + return answer; + }); + let opened: string | undefined; + const create = providerDouble(async () => { + if (when === 'lookup' && sibling === undefined) await siblingPushes(); + looked = true; + return opened; + }, async () => { + opened = 'https://github.com/fixture/app/pull/1'; + return opened; + }); + + expect(await leaseNext(config, runId, remote, { ...writable, ...(recorded ? {} : { running: [docs.attempt] }) })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + + if (!recorded) await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const message = `commit ${sibling}, which a later report recorded on the shared branch ${branch}, changed protected paths against the run base: .github/workflows/ci.yml.`; + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(message) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + it('records a pull request opened while the run was cancelled, keeps the step cancelled, and queues the end record again with it', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); config.usageReportEnabled = true; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index e40b87787..f7ecfef76 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -885,15 +885,16 @@ export interface NextOptions { /** * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still - * delivers `job`. If it does, the run lists its request in the same write, so one opened by a process that then stops - * is looked up, whatever becomes of the step, rather than lost. + * delivers `job`, and the branch's recorded commit is still the one `job` checked. If so, the run lists its request in + * the same write, so one opened by a process that then stops is looked up, whatever becomes of the step, rather than lost. */ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob): Promise<'open' | 'refused' | 'busy'> { - const marked = await updateRun(config, runId, async (run): Promise<{ value: 'open' | 'refused'; write?: RunState }> => { + const marked = await updateRun(config, runId, async (run): Promise<{ value: 'open' | 'refused' | 'busy'; write?: RunState }> => { if (run === 'missing' || run.status !== 'running') return { value: 'refused' }; const settledRun = reconcileRun(run); const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); if (!step) return { value: 'refused' }; + if (run.copies?.[job.key.branch]?.commit !== job.recordedHead) return { value: 'busy' }; const create = { ...job.key, target: job.target }; const listed = run.creating?.find((item) => sameCreate(item, create)); // A request listed before is asked again now, so its lookups count from this create. @@ -935,7 +936,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise markCreating(config, runId, job)); - // As when the pick is busy: the step still delivers, and a later call delivers it. + // As when the pick is busy: the step still delivers, and a later call checks and delivers it. if (outcome === 'busy') return requeued; if (!outcome.ok && outcome.fault.kind === 'branch_ahead') waiting.push(job.key); apply = (run) => completeDelivery(run, job, outcome, reconcileRun); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index ea74875ee..7349dd764 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -54,8 +54,9 @@ export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: Deli /** * Asked under the run lock right before a pull request is opened: `open` while the step still delivers the job, whose - * request the run then lists as unresolved; `refused` when it no longer does; `busy` when the run lock could not be - * taken, so nobody knows yet. Only `open` opens anything. + * request the run then lists as unresolved; `refused` when it no longer does; `busy` when nobody knows yet, because the + * run lock could not be taken or a report recorded another commit on the branch since the job was picked. Only `open` + * opens anything. */ export type CreateGate = () => Promise<'open' | 'refused' | 'busy'>; @@ -70,13 +71,13 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) /** * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch - * is looked up, and only when there is none, and `gate` opens, is one opened; then it is looked up again. `busy` is a - * gate that could not answer: nothing was opened, and the job is for a later call. + * is looked up, and only when there is none, origin's branch is still at the checked commit, and `gate` opens, is one + * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, or + * the gate could not answer, and the job is for a later call, which checks it again. * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { if (!OBJECT_ID.test(job.key.head) || !OBJECT_ID.test(job.base)) return failed({ kind: 'unreadable' }); - // A later push can still change an open request after this read; only the driver holding the branch can make it. const tip = await originTip(job.checkout, job.key.branch); if (tip.kind === 'unreadable') return failed({ kind: 'branch_unreadable' }); if (tip.commit === undefined) return failed({ kind: 'branch_missing' }); @@ -109,6 +110,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 20:23:10 +0200 Subject: [PATCH 65/77] fix(workflow): keep waiting for an ahead tip while a writer of the branch runs A step can push before it reports. Three `next` calls then failed the delivering step with `branch_moved` while the sibling's lease was still open, and its later report could not revive the delivery. A drain that finds origin's branch ahead of the recorded commit no longer counts toward the three-drain bound while another step that writes the same branch is leased. Red: "keeps waiting for a report of a commit ahead on origin while a step that writes the branch still runs, however many calls come" failed on the third `next`, which answered `wait.reason: "work.docs is leased"` without `delivering: ['work.impl']`, because the bound had failed `work.impl`. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 23 +++++++++++++++++++++++ src/workflow/delivery-ledger.ts | 13 +++++++++---- src/workflow/engine.ts | 14 +++++++++++++- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 96ff24c55..e2f45f6c4 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 76d2a09f2..a4bd14c25 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1074,6 +1074,29 @@ describe('retrying and cancelling a delivery', () => { expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); }); + it('keeps waiting for a report of a commit ahead on origin while a step that writes the branch still runs, however many calls come', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + + for (let drain = 1; drain <= 3; drain += 1) { + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + } + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(creates(gh)).toEqual([]); + await succeed(config, runId, docs, { to: ahead }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index facc10d52..8902aa372 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -81,13 +81,16 @@ function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrU * while origin's branch is ahead of the recorded commit and a report of it may still come. * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. */ -export function completeDelivery(run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, reconcile: (run: RunState) => unknown): { requeueEnd: boolean } { +export function completeDelivery( + run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, + reconcile: (run: RunState) => unknown, writerLeased: (run: RunState, branch: string) => boolean, +): { requeueEnd: boolean } { const receipted = outcome.ok && recordReceipt(run, job.key, job.target, outcome.pr); if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); if (run.status === 'running') reconcile(run); const step = deliveringStep(run, job.key); if (step && outcome.ok) delivered(step); - else if (step && !outcome.ok && !(outcome.fault.kind === 'branch_ahead' && awaitsReport(step, outcome.fault.tip))) { + else if (step && !outcome.ok && !(outcome.fault.kind === 'branch_ahead' && awaitsReport(step, outcome.fault.tip, writerLeased(run, job.key.branch)))) { deliveryFailed(step, deliveryFailure(job, outcome.fault)); } return { requeueEnd: receipted && run.status !== 'running' }; @@ -98,9 +101,11 @@ const AHEAD_DRAINS = 3; /** * Whether `step` still waits for a report of `tip`, the commit origin's branch is ahead at: one may be on its way, from a - * step of the same branch that pushed before it reported. The drain that finds it there a third time fails the step. + * step of the same branch that pushed before it reported. A drain counts only while no step that writes the branch is + * leased, and the one that finds it there a third time fails the step. */ -function awaitsReport(step: DeliveringStep, tip: string): boolean { +function awaitsReport(step: DeliveringStep, tip: string, writerLeased: boolean): boolean { + if (writerLeased) return true; const drains = step.unrecordedTip?.commit === tip ? step.unrecordedTip.drains + 1 : 1; if (drains >= AHEAD_DRAINS) return false; step.unrecordedTip = { commit: tip, drains }; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index f7ecfef76..cf0217a2f 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -939,7 +939,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise completeDelivery(run, job, outcome, reconcileRun); + apply = (run) => completeDelivery(run, job, outcome, reconcileRun, branchWriterLeased); } else { const { create, checkout } = picked.value; tried.push(create); @@ -1330,6 +1330,18 @@ function branchStillUsed(run: RunState, branch: string): boolean { return false; } +/** Whether a leased step writes `branch`, so a commit it pushed before it reported may already be on origin. */ +function branchWriterLeased(run: RunState, branch: string): boolean { + return run.steps.some((item) => { + if (item.status !== 'leased') return false; + const located = locateStep(run, item.id); + const plan = located && planWorkspace(run, located.scope, item, located.step, { + driver: item.lease.driver, modes: ['read-only', 'new', 'continue'], + }, true); + return plan !== undefined && plan.mode !== 'read-only' && plan.contract?.branch === branch; + }); +} + /** * The session chains the steps that have not settled will continue, once each: each chain is the session a step resumes * with the sessions it continued before, newest first. Like `branchStillUsed`, it reads the run as the next `next` settles From a97524ff9119b0cb597d0e3fbf0dfccc85a58157 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 20:36:05 +0200 Subject: [PATCH 66/77] fix(workflow): take a found pull request only once origin's branch is still at the checked commit A sibling can push while the lookup runs, and the lookup can find the branch's open request. Delivery then succeeded on the commit it checked before the lookup, though the request already held the sibling's unchecked one. The second read of origin's tip now comes before a found request is taken, as it does before a create. Red: 'opens nothing when a sibling pushes and records a protected commit while the lookup finds the branch's open pull request' failed with the run done instead of a wait for the delivery. Co-authored-by: Nathan Shan --- src/__tests__/workflow-delivery.test.ts | 11 ++++++----- src/workflow/pr-delivery.ts | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index a4bd14c25..6793d1450 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1253,10 +1253,11 @@ describe('retrying and cancelling a delivery', () => { }); it.each([ - ['pushes and records a protected commit while the lookup runs', 'lookup', true], - ['pushes a protected commit while the lookup runs, and records it later', 'lookup', false], - ['pushes and records a protected commit once delivery read origin\'s branch for the last time', 'last read', true], - ] as const)('opens nothing when a sibling %s, and checks that commit on the next call', async (_case, when, recorded) => { + ['pushes and records a protected commit while the lookup runs', 'lookup', true, false], + ['pushes a protected commit while the lookup runs, and records it later', 'lookup', false, false], + ['pushes and records a protected commit once delivery read origin\'s branch for the last time', 'last read', true, false], + ['pushes and records a protected commit while the lookup finds the branch\'s open pull request', 'lookup', true, true], + ] as const)('opens nothing when a sibling %s, and checks that commit on the next call', async (_case, when, recorded, open) => { const { config, root, runId } = await fixture(IMPL_AND_DOCS); const impl = await take(config, runId); await succeed(config, runId, impl, { to: pushWork(root, impl) }); @@ -1277,7 +1278,7 @@ describe('retrying and cancelling a delivery', () => { if (when === 'last read' && looked && sibling === undefined && args[1].includes('ls-remote')) await siblingPushes(); return answer; }); - let opened: string | undefined; + let opened: string | undefined = open ? 'https://github.com/fixture/app/pull/1' : undefined; const create = providerDouble(async () => { if (when === 'lookup' && sibling === undefined) await siblingPushes(); looked = true; diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 7349dd764..7bb7ade2b 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -107,14 +107,14 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 21:06:46 +0200 Subject: [PATCH 67/77] fix(workflow): open no pull request that leaves out work a step inside wrote A delivering workflow step with workspace: new delivers its own copy. A step inside it with workspace: new writes a branch of its own, and once another step recorded a commit on the copy, that branch was never part of the pull request, yet the workflow step succeeded. The job now carries the head of each step inside the delivering one that succeeded on another branch that no step inside it delivers. deliverPr requires each to be an ancestor of origin's tip, and otherwise fails with child_work_left_out, naming the step and its branch. A later step that merges that branch through needs makes the delivery pass. The run keeps the workflow it started with, so the remedy is a changed workflow and a new run. Red: "opens no pull request for a workflow step whose branch leaves out the work of a step inside it on another branch" failed with outcome "succeeded" instead of "failed". --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 33 ++++++++++++++++++++++++- src/workflow/delivery-ledger.ts | 4 +-- src/workflow/pr-delivery.ts | 14 +++++++++++ src/workflow/workspace.ts | 13 ++++++++++ 7 files changed, 64 insertions(+), 6 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index e2f45f6c4..4f113bd53 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index f43c21e54..3a2083c9e 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -218,7 +218,7 @@ steps: ## Deliver a PR -Set `deliver: pr` on a step that writes a branch, and TeamAI opens a pull request from that branch once the step succeeds. It never merges it. A step writes a branch when it has `workspace: new` or `continue`, or when it is a prompt inside a workflow that opened one. A command that delivers must set `workspace`, since a command without one is read-only. A `workflow` step with `workspace: new` delivers its branch once, for all its steps. On a `for_each` step, each item delivers its own branch. +Set `deliver: pr` on a step that writes a branch, and TeamAI opens a pull request from that branch once the step succeeds. It never merges it. A step writes a branch when it has `workspace: new` or `continue`, or when it is a prompt inside a workflow that opened one. A command that delivers must set `workspace`, since a command without one is read-only. A `workflow` step with `workspace: new` delivers its branch once, for all its steps. A step inside it that writes a branch of its own needs a later step that merges that branch through `needs`, or its own `deliver: pr`. Otherwise the delivery fails with `delivery_failed`, since the pull request would leave that work out. On a `for_each` step, each item delivers its own branch. ```yaml schemaVersion: v1alpha1 diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 5dbb7f015..33bc6e019 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -218,7 +218,7 @@ steps: ## 交付 PR -在写分支的步骤上设置 `deliver: pr`，步骤成功后 TeamAI 会从这条分支开一个 pull request，并且从不合并它。设置了 `workspace: new` 或 `continue` 的步骤会写分支，位于已开分支的工作流中的提示词步骤也会。交付的命令必须设置 `workspace`，因为没有 `workspace` 的命令是只读的。设置了 `workspace: new` 的 `workflow` 步骤为它的全部步骤只交付一次自己的分支。`for_each` 步骤的每个 item 各自交付自己的分支。 +在写分支的步骤上设置 `deliver: pr`，步骤成功后 TeamAI 会从这条分支开一个 pull request，并且从不合并它。设置了 `workspace: new` 或 `continue` 的步骤会写分支，位于已开分支的工作流中的提示词步骤也会。交付的命令必须设置 `workspace`，因为没有 `workspace` 的命令是只读的。设置了 `workspace: new` 的 `workflow` 步骤为它的全部步骤只交付一次自己的分支。其中写自己分支的步骤，需要有后续步骤通过 `needs` 合并那条分支，或者自己设置 `deliver: pr`。否则交付以 `delivery_failed` 失败，因为 pull request 会漏掉那部分工作。`for_each` 步骤的每个 item 各自交付自己的分支。 ```yaml schemaVersion: v1alpha1 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 6793d1450..ba0b5285e 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -371,7 +371,7 @@ describe('delivering a step', () => { const fix = await take(config, runId); if (!fix.workspace) throw new Error('fix has no workspace'); const to = pushWork(root, fix); - const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main' }; + const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [] }; expect(await deliverPr(job, async () => 'refused')).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); @@ -632,6 +632,37 @@ describe('where a pull request goes', () => { expect(run.deliveries?.map((receipt) => [receipt.step, receipt.branch])).toEqual([['a', a], ['change', change]]); }); + /** A workflow step that delivers its copy, where `a` writes the copy and `b` writes a branch of its own after it. */ + const COPY_AND_SIDE_BRANCH = [ + ` - id: work\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: a\n workspace: continue\n command: ${bump('a.txt')}\n`, + ` - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n`, + ].join(''); + + it('opens no pull request for a workflow step whose branch leaves out the work of a step inside it on another branch', async () => { + const { config, cwd, root, runId } = await fixture(COPY_AND_SIDE_BRANCH); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const [work, b] = [`teamai/${runId}/work/p1`, `teamai/${runId}/work-b/p1`]; + expect(run.steps.find((step) => step.id === 'work.b')).toMatchObject({ status: 'succeeded', changes: { branch: b } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`step "work.b" wrote branch ${b}, which branch ${work} does not hold, so a pull request from it would leave that work out`) } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ failure: { message: expect.stringContaining('Have a later step in the workflow need step "work.b", so it merges branch') } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + }); + + it('opens the pull request of a workflow step once a later step inside it merges the branch another step wrote', async () => { + const { config, cwd, root, runId } = await fixture(`${COPY_AND_SIDE_BRANCH} - id: c\n needs: [b]\n workspace: continue\n command: ${bump('c.txt')}\n`); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const work = `teamai/${runId}/work/p1`; + expect(creates(gh)).toEqual([[work, 'main']]); + expect(commitsBeyond(root, 'main', work)).toContain(changedTo(await state(config, runId), 'work.b')); + }); + it('opens a pull request for each pass of a looped new step, leaves the earlier ones open, and lists them in order', async () => { const { config, cwd, root, runId } = await fixture( ` - id: fix\n workspace: new\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 2", max_iterations: 2 }\n`, diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index 8902aa372..e8df1647a 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -4,7 +4,7 @@ import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import { deliveryFailure, type DeliveryJob, type DeliveryOutcome, type findDelivered } from './pr-delivery.js'; import { delivered, deliveryFailed, type DeliveringStep } from './step-state.js'; import type { RunState } from './types.js'; -import { deliveryTarget } from './workspace.js'; +import { childWorkOff, deliveryTarget } from './workspace.js'; /** The delivery of `step`, as the drain and the outcome it applies name it. */ function deliveryKey(step: DeliveringStep): DeliveryKey { @@ -41,7 +41,7 @@ export function nextDelivery(run: RunState, reconcile: (run: RunState) => object } return { key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, - recordedHead: run.copies?.[key.branch]?.commit, + recordedHead: run.copies?.[key.branch]?.commit, childWork: childWorkOff(run, step, key.branch), ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 7bb7ade2b..8450ee71c 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -23,6 +23,8 @@ export interface DeliveryJob { base: string; /** The latest commit a report recorded on this branch, read under the run lock. */ recordedHead?: string; + /** Work steps inside the delivering one left on branches no step delivers, which the delivered branch must hold. */ + childWork: Array<{ step: string; branch: string; commit: string }>; /** The branch the pull request goes into. */ target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ @@ -41,6 +43,8 @@ export type DeliveryFault = | { kind: 'branch_ahead'; tip: string } /** The recorded commit origin's branch is at lacks the step's head: a later step rewrote the branch's history. */ | { kind: 'branch_rewritten' } + /** Origin's branch lacks the work step `step` inside the delivering one left on `branch`. */ + | { kind: 'child_work_left_out'; step: string; branch: string } | { kind: 'branch_missing' } | { kind: 'branch_unreadable' } | { kind: 'no_provider' } @@ -101,6 +105,11 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise 0 ? { kind: 'protected_path', files: owned } : { kind: 'later_protected_path', files }); } if (files.length > 0) return failed({ kind: 'protected_path', files }); + for (const child of job.childWork) { + const held = await contains(job, { commit: tip.commit, ancestor: child.commit }); + if ('fault' in held) return failed(held.fault); + if (!held.contains) return failed({ kind: 'child_work_left_out', step: child.step, branch: child.branch }); + } if (job.reported) return { ok: true, pr: job.reported }; const origin = await originOf(job.checkout); if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); @@ -270,6 +279,11 @@ export function deliveryFailure({ runId, key, recordedHead }: Pick { + const inside = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status === 'succeeded'); + const delivered = new Set(inside.flatMap((item) => ('delivery' in item && item.delivery ? [item.delivery.branch] : []))); + return inside.flatMap((item) => { + const changes = changesOf(item); + return changes && changes.branch !== branch && !delivered.has(changes.branch) ? [{ step: item.id, branch: changes.branch, commit: changes.to }] : []; + }); +} + /** * The branch a pull request from `branch` goes into: the nearest branch it grew from that delivered a pull request, so * the request holds only the work done since, else the run base branch. Undefined when the run started from no branch. From d0438049f4c38b5faf1228ff26cbbf447a45fdc4 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 21:13:37 +0200 Subject: [PATCH 68/77] fix(workflow): drop a reported URL on the origin host that cannot be checked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A URL a driver reported on the origin's own host was kept when the lookup that checks it failed or answered a URL TeamAI cannot keep, so an unrelated or nonexistent pull request URL stayed on the step and was delivered as it was. On the origin's host, such a lookup now drops the URL with the reason "could not be checked against the open pull requests of branch …", which never names the URL. Delivery's own lookup then finds the open pull request. A URL on another host is still kept as it is. The driver conformance case for an unreachable host now expects a reporting driver to fail delivery as the others do. Red: "drops a URL on the origin host when the lookup that checks it cannot answer" and "... answers with a URL on another host" failed with no dropped entry: the URL was kept. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 30 ++++++++++++++++++++----- src/workflow/conformance.ts | 2 +- src/workflow/pr-delivery.ts | 9 ++++---- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 4f113bd53..ed4c1defb 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index 3a2083c9e..b016311ba 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -248,7 +248,7 @@ If origin has no branch to deliver, TeamAI opens no request. The failure names t If a host may have opened the request but TeamAI cannot confirm it, the step fails with `delivery_failed` and records no URL. Run the retry command in the message. TeamAI looks for the request before opening one again. -A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target, and one TeamAI cannot check with the host is dropped. Delivery then finds the open pull request itself. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. ### Start from CI diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 33bc6e019..4f8a037df 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -248,7 +248,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 如果主机可能已打开请求，但 TeamAI 无法确认，步骤会以 `delivery_failed` 失败，且不记录 URL。运行消息中的重试命令。TeamAI 会先查找已有请求，再尝试打开请求。 -自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request；TeamAI 无法向主机核对的 URL 会被丢弃，之后由交付自己查找已打开的 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 ### 从 CI 启动 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index ba0b5285e..e3993e89a 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -14,7 +14,7 @@ import * as store from '../workflow/store.js'; import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; import { acquireLock, releaseLock } from '../update.js'; import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; -import { fakeGh } from './helpers/fake-gh.js'; +import { fakeGh, type FakeGhOptions } from './helpers/fake-gh.js'; import { deliverPr } from '../workflow/pr-delivery.js'; import { buildEndRecord, pendingRecords } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; @@ -837,13 +837,10 @@ describe('reported URLs from a driver', () => { expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual([`https://github.com/fixture/app/pull/${delivered}`]); }); - it.each([ - ['is the open pull request of its branch', {}], - ['cannot be checked while GitHub cannot be reached', { lookup: 'fail' as const }], - ])('keeps a URL on the origin host that %s, and opens nothing for it', async (_case, options) => { + it('keeps a URL on the origin host that is the open pull request of its branch, and opens nothing for it', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); - const gh = fakeGh(path.join(root, 'gh'), { ...options, open: [pull(4, `teamai/${runId}/fix/p1`)] }); + const gh = fakeGh(path.join(root, 'gh'), { open: [pull(4, `teamai/${runId}/fix/p1`)] }); const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/4?tab=files#top' }); expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'fix', status: 'delivering' } } }); expect(accepted).not.toHaveProperty('data.dropped'); @@ -852,6 +849,27 @@ describe('reported URLs from a driver', () => { expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/4']); expect(gh.calls().filter((call) => call.args[0] === 'pr')).toEqual([]); }); + + it.each([ + ['cannot answer', (): FakeGhOptions => ({ lookup: 'fail' })], + ['answers with a URL on another host', (branch: string): FakeGhOptions => ({ open: [{ ...pull(4, branch), html_url: 'https://github.example.net/fixture/app/pull/4' }] })], + ])('drops a URL on the origin host when the lookup that checks it %s, without its value, and delivers the open one later', async (_case, options) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), options(branch)); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/4' }); + expect(accepted).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: `could not be checked against the open pull requests of branch ${branch} into main` }], + } }); + expect(JSON.stringify(accepted)).not.toContain('pull/4'); + expect((await state(config, runId)).steps[0]).not.toHaveProperty('delivery.reported'); + + gh.configure({ open: [pull(4, branch)] }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/4']); + expect(creates(gh)).toEqual([]); + }); }); /** diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index ee258be5e..4b6d2d973 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -988,7 +988,7 @@ export const workflowDeliveryConformanceScenarios: DeliveryConformanceScenario[] name: 'a delivery that cannot reach the host fails naming the branch, which stays on origin', unreachable: true, opened: { creates: 0, run: 'failed', work: 'failed', prs: 0, failure: 'delivery_failed' }, - reported: REPORTED, + reported: { creates: 0, run: 'failed', work: 'failed', prs: 0, failure: 'delivery_failed' }, }, { name: 'a driver polls while another process of its machine delivers, and gets the step that needs it once delivered', diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 8450ee71c..77c059ca1 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -150,8 +150,9 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 21:20:07 +0200 Subject: [PATCH 69/77] fix(workflow): name remedies a protected later commit and an ahead tip accept later_protected_path told the person to have a step on the branch take the changes out. In the topology it exists for, the step that pushed them already succeeded, so no such step is left, and a hand-pushed removal makes origin's tip an ahead tip, which delivery rejects too. The message now names the two remedies the run accepts: put origin's branch back at the step's own commit and retry the step, which delivers it without the later commits, or retry the workflow step whose copy the branch is, which runs its steps again in a new branch. The job carries that workflow step as openedBy. branch_ahead no longer reuses the branch_moved text, which said the tip no longer holds the checked changes. Its own message says the tip holds the recorded commit and later commits no report recorded. Red: "names remedies for a protected change a later report recorded on the shared branch" failed with the old "Have a step on that branch take those changes out" message, and "opens and accepts nothing when origin's branch moved past the checked commit" failed with the branch_moved text. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 32 +++++++++++++++++++++++-- src/workflow/delivery-ledger.ts | 3 ++- src/workflow/pr-delivery.ts | 16 +++++++++---- 6 files changed, 47 insertions(+), 10 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index ed4c1defb..10c3683e3 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_moved` message. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index b016311ba..c3634ad12 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -240,7 +240,7 @@ A pull request goes into the nearest branch its own branch grew from that alread Each pass of a looped `workspace: new` step has its own `/p` branch, so it opens its own pull request. The earlier ones stay open, by design. A step that loops on one `continue` branch has one pull request. -Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. When only a later commit that another step recorded on a shared branch made the change, the step fails with `delivery_failed` instead, and the retry does not run it again. +Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. When only a later commit that another step recorded on a shared branch made the change, the step fails with `delivery_failed` instead, and the retry does not run it again. Its message says how to deliver only the step's own commit, or how to run the steps of that branch again in a new one. Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 4f8a037df..f33c4bb7e 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -240,7 +240,7 @@ pull request 合入这条分支来源中最近一条已经交付过的分支， 循环中 `workspace: new` 步骤的每个 pass 都有自己的 `/p` 分支，因此各开一个 pull request。之前的 pull request 保持打开，这是有意为之。在同一条 `continue` 分支上循环的步骤只有一个 pull request。 -开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。如果修改只来自另一个步骤之后在共享分支上记录的提交，步骤改为以 `delivery_failed` 失败，重试也不会重新运行它。 +开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。如果修改只来自另一个步骤之后在共享分支上记录的提交，步骤改为以 `delivery_failed` 失败，重试也不会重新运行它。消息会说明如何只交付该步骤自己的提交，或如何在新分支上重新运行那条分支的步骤。 其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index e3993e89a..d5e177e5b 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -422,14 +422,14 @@ describe('delivering a step', () => { const branch = `teamai/${runId}/fix/p1`; const gh = fakeGh(path.join(root, 'gh'), { open: 'pr_url' in changes ? [pull(1, branch)] : [] }); const checked = pushWork(root, fix, { 'src/app.ts': 'ok\n' }); - pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + const ahead = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); expect(await succeed(config, runId, fix, { to: checked, ...changes })).not.toHaveProperty('data.dropped'); for (const _drain of [1, 2]) expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['fix'] } } }); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const run = await state(config, runId); expect(run.steps).toMatchObject([ - { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin moved away from commit ${checked}`) } }, + { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin is at commit ${ahead}, which holds commit ${checked}, the last one a report recorded on it, and later commits no report recorded, so TeamAI has not checked them`) } }, { id: 'review', status: 'skipped' }, ]); expect(run.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); @@ -998,6 +998,34 @@ describe('retrying and cancelling a delivery', () => { } }); + it.each(['puts origin back at the step\'s own commit', 'retries the workflow step'] as const)('names remedies for a protected change a later report recorded on the shared branch, and the run accepts the one that %s', async (remedy) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const docs = await take(config, runId); + const branch = docs.workspace?.branch ?? ''; + await succeed(config, runId, docs, { to: pushOnto(root, branch, { '.github/workflows/docs.yml': 'docs' }) }); + gh.configure({}); + await retryStep(config, runId, 'work.impl'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(`put branch ${branch} on origin back at ${own}, the commit the step reported, then run: teamai workflow retry ${runId} work.impl`) } }); + expect(failure).toMatchObject({ failure: { message: expect.stringContaining(`run its steps again in a new branch: teamai workflow retry ${runId} work`) } }); + + if (remedy === 'retries the workflow step') { + expect(await retryStep(config, runId, 'work')).toMatchObject({ data: { retried: { step: 'work', pass: 2 } } }); + expect(await take(config, runId)).toMatchObject({ step: 'work.impl', workspace: { branch: `teamai/${runId}/work/p2` } }); + return; + } + git(root, ['--git-dir=remote.git', 'update-ref', `refs/heads/${branch}`, own]); + await retryStep(config, runId, 'work.impl'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + it('blames the delivering step for its own protected change when a later report recorded more work on the shared branch', async () => { const { config, root, runId } = await fixture(IMPL_AND_DOCS); const gh = fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index e8df1647a..2f4c74e5f 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -4,7 +4,7 @@ import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import { deliveryFailure, type DeliveryJob, type DeliveryOutcome, type findDelivered } from './pr-delivery.js'; import { delivered, deliveryFailed, type DeliveringStep } from './step-state.js'; import type { RunState } from './types.js'; -import { childWorkOff, deliveryTarget } from './workspace.js'; +import { branchName, childWorkOff, deliveryTarget } from './workspace.js'; /** The delivery of `step`, as the drain and the outcome it applies name it. */ function deliveryKey(step: DeliveringStep): DeliveryKey { @@ -42,6 +42,7 @@ export function nextDelivery(run: RunState, reconcile: (run: RunState) => object return { key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, recordedHead: run.copies?.[key.branch]?.commit, childWork: childWorkOff(run, step, key.branch), + openedBy: run.steps.find((item) => item !== step && branchName(run.id, item.id, item.pass ?? 1) === key.branch)?.id, ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 77c059ca1..861943bf5 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -25,6 +25,8 @@ export interface DeliveryJob { recordedHead?: string; /** Work steps inside the delivering one left on branches no step delivers, which the delivered branch must hold. */ childWork: Array<{ step: string; branch: string; commit: string }>; + /** The workflow step whose copy the branch is, when another step's: retrying it runs its steps again in a new branch. */ + openedBy?: string; /** The branch the pull request goes into. */ target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ @@ -239,7 +241,7 @@ function listedFiles(files: string[]): string { } /** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ -export function deliveryFailure({ runId, key, recordedHead }: Pick, fault: DeliveryFault): DeliveryFailure { +export function deliveryFailure({ runId, key, recordedHead, openedBy }: Pick, fault: DeliveryFault): DeliveryFailure { const { step, branch } = key; const head = recordedHead ?? key.head; const retry = `teamai workflow retry ${runId} ${step}`; @@ -259,18 +261,24 @@ export function deliveryFailure({ runId, key, recordedHead }: Pick Date: Sat, 24 Oct 2026 21:26:04 +0200 Subject: [PATCH 70/77] fix(workflow): fail an ahead tip once no report of it can come A tip ahead of the recorded commit kept its step delivering for three drains in a row, counted in the step's persisted unrecordedTip. Since a writer lease already holds the wait, the count only waited on driver polls during which nothing could record the tip. completeDelivery now keeps the step delivering while a step that writes the branch is leased, or when copies[branch].commit is no longer the job's recordedHead, which a report that landed while the drain read the branch changes. Otherwise it fails the step at once. unrecordedTip, its schema member, snapshot rule and destructurings, AHEAD_DRAINS and awaitsReport are gone. The per-drain waiting list stays. Red: "opens and accepts nothing when origin's branch moved past the checked commit" (both cases) and "names the commit the run recorded last when the shared branch on origin is moved at a retry" failed with a wait answer instead of the failed run. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 35 +++++++++++++++++++++---- src/workflow/delivery-ledger.ts | 27 +++++-------------- src/workflow/snapshot.ts | 1 - src/workflow/step-state.ts | 12 +++------ 5 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 10c3683e3..66b607c3f 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`, since a report of it may still be on its way: the step stays `delivering`, the drain does not pick it again, and `completeDelivery` counts in the step's `unrecordedTip` the drains in a row that found that tip, failing the step on the third with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. A drain does not count while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports. The snapshot drops the count. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index d5e177e5b..e9b30d24d 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -425,7 +425,7 @@ describe('delivering a step', () => { const ahead = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); expect(await succeed(config, runId, fix, { to: checked, ...changes })).not.toHaveProperty('data.dropped'); - for (const _drain of [1, 2]) expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['fix'] } } }); + // No step that writes the branch is leased, so no report can record that commit, and the first call fails the step. expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const run = await state(config, runId); expect(run.steps).toMatchObject([ @@ -1119,10 +1119,6 @@ describe('retrying and cancelling a delivery', () => { gh.configure({}); await retryStep(config, runId, 'work.impl'); - const drains = change === 'moved' ? 3 : 1; - for (let drain = 1; drain < drains; drain += 1) { - expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); - } expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); const remedy = change === 'moved' ? `Put branch ${branch} on origin back at ${recorded}, then run` : `push ${recorded} to it, then run`; @@ -1174,6 +1170,35 @@ describe('retrying and cancelling a delivery', () => { expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); }); + it('keeps a step delivering when a report records origin\'s ahead commit while delivery reads the branch, and delivers it on the next call', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + let reported = false; + const read = worktree.git; + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + const answer = await read(...args); + if (!reported && args[1].includes('ls-remote')) { + reported = true; + await succeed(config, runId, docs, { to: ahead }); + } + return answer; + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + expect(reported).toBe(true); + expect(creates(gh)).toEqual([]); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + }); + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); const fix = await take(config, runId); diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index 2f4c74e5f..9132f5228 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -78,8 +78,9 @@ function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrU * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The - * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it, and not - * while origin's branch is ahead of the recorded commit and a report of it may still come. + * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. Origin's + * branch ahead of the recorded commit fails it only once no report of that tip can come: no step that writes the branch + * is leased, and no report recorded another commit on it since the job was picked. * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. */ export function completeDelivery( @@ -90,29 +91,13 @@ export function completeDelivery( if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); if (run.status === 'running') reconcile(run); const step = deliveringStep(run, job.key); + const awaited = !outcome.ok && outcome.fault.kind === 'branch_ahead' + && (writerLeased(run, job.key.branch) || run.copies?.[job.key.branch]?.commit !== job.recordedHead); if (step && outcome.ok) delivered(step); - else if (step && !outcome.ok && !(outcome.fault.kind === 'branch_ahead' && awaitsReport(step, outcome.fault.tip, writerLeased(run, job.key.branch)))) { - deliveryFailed(step, deliveryFailure(job, outcome.fault)); - } + else if (step && !outcome.ok && !awaited) deliveryFailed(step, deliveryFailure(job, outcome.fault)); return { requeueEnd: receipted && run.status !== 'running' }; } -/** How many drains in a row may find origin's branch ahead of the recorded commit, at one commit, before delivery fails. */ -const AHEAD_DRAINS = 3; - -/** - * Whether `step` still waits for a report of `tip`, the commit origin's branch is ahead at: one may be on its way, from a - * step of the same branch that pushed before it reported. A drain counts only while no step that writes the branch is - * leased, and the one that finds it there a third time fails the step. - */ -function awaitsReport(step: DeliveringStep, tip: string, writerLeased: boolean): boolean { - if (writerLeased) return true; - const drains = step.unrecordedTip?.commit === tip ? step.unrecordedTip.drains + 1 : 1; - if (drains >= AHEAD_DRAINS) return false; - step.unrecordedTip = { commit: tip, drains }; - return true; -} - /** * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index d88955ff3..c7e3bb437 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -205,7 +205,6 @@ const STEP_RULES = { outputs: 'publish', end: 'publish', results: 'publish', settledAt: 'publish', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', delivery: 'publish', - unrecordedTip: 'drop', } satisfies FieldRules; const PASS_RULES = { diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index 76bfc04e6..37fd27f03 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -162,11 +162,7 @@ const PendingSchema = StepBaseSchema.extend({ status: z.literal('pending') }); const WaitingSchema = StepBaseSchema.extend({ status: z.literal('waiting') }); const RunningSchema = StepBaseSchema.extend({ status: z.literal('running') }); /** `delivering`: the agent succeeded and its work is on its branch; dependents wait until that branch has its pull request. */ -const DeliveringSchema = StepBaseSchema.extend({ - status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema, - /** The commit origin's branch was found ahead at, past the recorded one, and how many drains in a row found it there. */ - unrecordedTip: z.object({ commit: z.string(), drains: z.number().int().positive() }).optional(), -}); +const DeliveringSchema = StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema }); const SucceededSchema = StepBaseSchema.extend({ status: z.literal('succeeded'), outcome: z.literal('success').default('success'), ...settledFields, loopFinal: z.boolean().optional(), }); @@ -302,7 +298,7 @@ export function startDelivery(state: StepState, settlement: Settlement): Fail */ export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { if (state.status === 'delivering') { - const { id: _id, status: _status, unrecordedTip: _tip, ...settlement } = state; + const { id: _id, status: _status, ...settlement } = state; return become(state, { ...settlement, status: 'cancelled', failure }); } const attempt = state.status === 'leased' ? state.attempt : undefined; From 354fcedc0b2d03926a05a078cab4cee572b29769 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 21:33:35 +0200 Subject: [PATCH 71/77] fix(workflow): deliver nothing while another writer of the branch is leased The gate opened a pull request, and a reported URL was accepted, while a sibling that writes the same branch was still leased. That sibling's next push then entered the pull request with no delivery checking it. markCreating now answers busy while branchWriterLeased is true. The job carries writerLeased, read under the run lock when it is picked, and a reported URL or a request the lookup found is busy rather than accepted while it is set. Busy leaves the step delivering, so the drain after the sibling reports checks the sibling's commit. Red: the three cases of "delivers nothing while a sibling that writes the shared branch is leased" failed with wait.reason "work.docs is leased" and no delivering list: the step had already delivered. --- docs/dev/workflow-engine.md | 2 +- docs/guide/workflow.md | 2 +- docs/guide/zh-CN/workflow.md | 2 +- src/__tests__/workflow-delivery.test.ts | 32 ++++++++++++++++++++++++- src/workflow/delivery-ledger.ts | 5 +++- src/workflow/engine.ts | 9 +++---- src/workflow/pr-delivery.ts | 14 ++++++----- 7 files changed, 51 insertions(+), 15 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 66b607c3f..446b4680d 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead`, and lists the request in `run.creating`. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. A reported URL or a request the lookup found is `busy` instead of accepted while the job's `writerLeased`, read under the run lock when the job was picked, says such a step was leased, so the next drain checks that step's commit. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c3634ad12..f72b1cd9b 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -234,7 +234,7 @@ steps: A `deliver: pr` step that would run read-only fails validation, and so does a delivering `workflow` step, `for_each` or not, when no step of its workflow writes a branch. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. -When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. A run that a crash stopped finds the pull request it opened before and opens no other. So does a run another machine took over, unless the machine that handed it over is still opening one at that moment. GitHub and GitLab reject such a duplicate. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. +When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. While another step that writes the same branch still runs, delivery waits for its report, so TeamAI checks that step's commit before the pull request is opened or taken. A run that a crash stopped finds the pull request it opened before and opens no other. So does a run another machine took over, unless the machine that handed it over is still opening one at that moment. GitHub and GitLab reject such a duplicate. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. A pull request goes into the nearest branch its own branch grew from that already delivered one, or else into the branch the run started from. A branch grows first from its `for_each` item's `after` branches, then from its `needs` in order. The first step of a sub-workflow grows from the needs of the step that runs it. A skipped step wrote no branch, so it is passed over. Each pull request then shows only the work done since the branch it goes into. diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index f33c4bb7e..744252683 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -234,7 +234,7 @@ steps: 会以只读方式运行的 `deliver: pr` 步骤无法通过校验；设置了 `deliver: pr` 的 `workflow` 步骤（无论是否 `for_each`），若其工作流中没有步骤写分支，也无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 -步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。因崩溃而中断的运行，会找到之前开的 pull request，不会再开第二个。被另一台机器接管的运行也是如此，除非交出运行的机器此刻仍在开 pull request；GitHub 和 GitLab 会拒绝这样的重复请求。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 +步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。写同一条分支的另一个步骤仍在运行时，交付会等它上报，这样 TeamAI 会在开出或接受 pull request 之前检查那个步骤的提交。因崩溃而中断的运行，会找到之前开的 pull request，不会再开第二个。被另一台机器接管的运行也是如此，除非交出运行的机器此刻仍在开 pull request；GitHub 和 GitLab 会拒绝这样的重复请求。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 pull request 合入这条分支来源中最近一条已经交付过的分支，没有时合入运行开始时的分支。分支首先来自其 `for_each` item 的 `after` 分支，然后依次来自 `needs`。子工作流的第一个步骤来自运行它的步骤的 needs。跳过的步骤没有写分支，会被越过。因此每个 pull request 只显示它所合入的分支之后的工作。 diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index e9b30d24d..fe765e4b4 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -371,7 +371,7 @@ describe('delivering a step', () => { const fix = await take(config, runId); if (!fix.workspace) throw new Error('fix has no workspace'); const to = pushWork(root, fix); - const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [] }; + const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [], writerLeased: false }; expect(await deliverPr(job, async () => 'refused')).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); @@ -1405,6 +1405,36 @@ describe('retrying and cancelling a delivery', () => { expect(create).not.toHaveBeenCalled(); }); + it.each([ + ['TeamAI would open its pull request', {}, false], + ['its driver reported its pull request', { pr_url: 'https://github.com/fixture/app/pull/1' }, true], + ['the lookup finds its open pull request', {}, true], + ])('delivers nothing while a sibling that writes the shared branch is leased, when %s, and checks the sibling\'s commit once it reports', async (_case, changes, open) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + const branch = impl.workspace?.branch ?? ''; + const gh = fakeGh(path.join(root, 'gh'), { open: open ? [pull(1, branch)] : [] }); + expect(await succeed(config, runId, impl, { to: pushWork(root, impl), ...changes })).not.toHaveProperty('data.dropped'); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(waiting.deliveries).toBeUndefined(); + + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(creates(gh)).toEqual([]); + }); + it('records a pull request opened while the run was cancelled, keeps the step cancelled, and queues the end record again with it', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); config.usageReportEnabled = true; diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index 9132f5228..3bf46213f 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -26,7 +26,9 @@ export function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep * stopped is not delivered. One in `waiting`, whose branch this drain found ahead of the recorded commit, waits for a * later drain. Undefined when nothing delivers, or when the run settles, which the locked body of `next` finishes. */ -export function nextDelivery(run: RunState, reconcile: (run: RunState) => object, waiting: readonly DeliveryKey[]): DeliveryJob | undefined { +export function nextDelivery( + run: RunState, reconcile: (run: RunState) => object, writerLeased: (run: RunState, branch: string) => boolean, waiting: readonly DeliveryKey[], +): DeliveryJob | undefined { if (run.status !== 'running') return undefined; const settledRun = reconcile(run); if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; @@ -43,6 +45,7 @@ export function nextDelivery(run: RunState, reconcile: (run: RunState) => object key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, recordedHead: run.copies?.[key.branch]?.commit, childWork: childWorkOff(run, step, key.branch), openedBy: run.steps.find((item) => item !== step && branchName(run.id, item.id, item.pass ?? 1) === key.branch)?.id, + writerLeased: writerLeased(run, key.branch), ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index cf0217a2f..26f8f522f 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -885,8 +885,9 @@ export interface NextOptions { /** * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still - * delivers `job`, and the branch's recorded commit is still the one `job` checked. If so, the run lists its request in - * the same write, so one opened by a process that then stops is looked up, whatever becomes of the step, rather than lost. + * delivers `job`, the branch's recorded commit is still the one `job` checked, and no step that writes the branch is + * leased. If so, the run lists its request in the same write, so one opened by a process that then stops is looked up, + * whatever becomes of the step, rather than lost. */ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob): Promise<'open' | 'refused' | 'busy'> { const marked = await updateRun(config, runId, async (run): Promise<{ value: 'open' | 'refused' | 'busy'; write?: RunState }> => { @@ -894,7 +895,7 @@ async function markCreating(config: LocalConfig, runId: string, job: DeliveryJob const settledRun = reconcileRun(run); const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); if (!step) return { value: 'refused' }; - if (run.copies?.[job.key.branch]?.commit !== job.recordedHead) return { value: 'busy' }; + if (run.copies?.[job.key.branch]?.commit !== job.recordedHead || branchWriterLeased(run, job.key.branch)) return { value: 'busy' }; const create = { ...job.key, target: job.target }; const listed = run.creating?.find((item) => sameCreate(item, create)); // A request listed before is asked again now, so its lookups count from this create. @@ -926,7 +927,7 @@ async function drainDeliveries(config: LocalConfig, runId: string): Promise { if (run === 'missing') return { value: undefined }; const before = JSON.stringify(run); - const job = nextDelivery(run, reconcileRun, waiting); + const job = nextDelivery(run, reconcileRun, branchWriterLeased, waiting); const value = job ? { job } : unresolvedCreate(run, tried); return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: new Date().toISOString() } } : {}) }; }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 861943bf5..67548e194 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -27,6 +27,8 @@ export interface DeliveryJob { childWork: Array<{ step: string; branch: string; commit: string }>; /** The workflow step whose copy the branch is, when another step's: retrying it runs its steps again in a new branch. */ openedBy?: string; + /** A step that writes the branch was leased, so its next push would enter a pull request unchecked: none is taken yet. */ + writerLeased: boolean; /** The branch the pull request goes into. */ target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ @@ -61,8 +63,8 @@ export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: Deli /** * Asked under the run lock right before a pull request is opened: `open` while the step still delivers the job, whose * request the run then lists as unresolved; `refused` when it no longer does; `busy` when nobody knows yet, because the - * run lock could not be taken or a report recorded another commit on the branch since the job was picked. Only `open` - * opens anything. + * run lock could not be taken, a report recorded another commit on the branch since the job was picked, or a step that + * writes the branch is leased. Only `open` opens anything. */ export type CreateGate = () => Promise<'open' | 'refused' | 'busy'>; @@ -78,8 +80,8 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch * is looked up, and only when there is none, origin's branch is still at the checked commit, and `gate` opens, is one - * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, or - * the gate could not answer, and the job is for a later call, which checks it again. + * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, a + * step that writes it was leased, or the gate could not answer, and the job is for a later call, which checks it again. * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { @@ -112,7 +114,7 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 21:38:41 +0200 Subject: [PATCH 72/77] fix(workflow): bound the fetch delivery runs, and keep it from prompting Delivery bounds and silences its ls-remote, but the fetch it runs when the checked tip or the recorded commit is missing locally had neither a timeout nor a non-interactive environment. A stalled connection after ls-remote held the delivery lock, and leaseNext with it, without limit. fetchMissingCommits takes an optional bound and environment for its fetch. Delivery passes DELIVERY_TIMEOUT_MS and NON_INTERACTIVE_GIT_ENV, as it does for ls-remote. The local driver's callers are unchanged. Red: "fetches the commits it checks within the delivery timeout and without credential prompts" failed because fetchMissingCommits was called with no options, and the fetch's git process got the GIT_ASKPASS of the member's environment. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 23 +++++++++++++++++++++++ src/workflow/pr-delivery.ts | 9 ++++++--- src/workflow/worktree.ts | 8 +++++--- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 446b4680d..8cf8db41a 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. A reported URL or a request the lookup found is `busy` instead of accepted while the job's `writerLeased`, read under the run lock when the job was picked, says such a step was leased, so the next drain checks that step's commit. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. A reported URL or a request the lookup found is `busy` instead of accepted while the job's `writerLeased`, read under the run lock when the job was picked, says such a step was leased, so the next drain checks that step's commit. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index fe765e4b4..8e43c35c8 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -458,6 +458,29 @@ describe('delivering a step', () => { expect(gh.calls()).toEqual([]); }); + it('fetches the commits it checks within the delivery timeout and without credential prompts', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + // Every git process that reaches origin logs the askpass it was given, then serves the fixture's bare repository. + const [log, transport, bare] = [path.join(root, 'transport.log'), path.join(root, 'transport.sh'), path.join(root, 'remote.git')]; + fs.writeFileSync(transport, `#!/bin/sh\nprintf '%s\\n' "$GIT_ASKPASS" >> '${log}'\nexec git upload-pack "$1"\n`, { mode: 0o755 }); + git(cwd, ['config', '--unset', `url.${bare}.insteadOf`]); + git(cwd, ['config', `url.ext::${transport} ${bare}.insteadOf`, 'https://github.com/fixture/app.git']); + git(cwd, ['config', 'protocol.ext.allow', 'always']); + vi.stubEnv('GIT_ASKPASS', '/opt/editor/askpass.sh'); + const fetches = vi.spyOn(worktree, 'fetchMissingCommits'); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const env = { GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo', GCM_INTERACTIVE: 'never' }; + expect(fetches).toHaveBeenCalledWith(cwd, { run: runId, commits: [fix.workspace?.base.commit, to] }, { timeoutMs: 30000, env }); + const askpass = fs.readFileSync(log, 'utf8').trim().split('\n'); + expect(askpass.length).toBeGreaterThanOrEqual(3); + expect(askpass.filter((given) => given !== 'echo')).toEqual([]); + }); + it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { const { config, root, runId } = await fixture(FIX_THEN_REVIEW); fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index 67548e194..b1df87d79 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -71,6 +71,9 @@ export type CreateGate = () => Promise<'open' | 'refused' | 'busy'>; /** How long a host may take to answer one lookup or one create. */ export const DELIVERY_TIMEOUT_MS = 30_000; +/** How delivery runs git that reaches origin: bounded, and never waiting on a prompt. */ +const ORIGIN_GIT = { timeoutMs: DELIVERY_TIMEOUT_MS, env: NON_INTERACTIVE_GIT_ENV }; + /** A full object id: the head comes from a driver's report, and anything else could reach git as an option. */ const OBJECT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/i; @@ -208,7 +211,7 @@ async function findOpen( /** The commit origin's `branch` is at, read by the checkout; none when origin has no such branch. */ async function originTip(checkout: string, branch: string): Promise<{ kind: 'read'; commit?: string } | { kind: 'unreadable' }> { const ref = `refs/heads/${branch}`; - const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], { timeoutMs: DELIVERY_TIMEOUT_MS, env: NON_INTERACTIVE_GIT_ENV }); + const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], ORIGIN_GIT); if (listed.code !== 0) return { kind: 'unreadable' }; const commit = listed.stdout.split('\n').map((line) => line.split('\t')).find(([, name]) => name?.trim() === ref)?.[0]; return commit ? { kind: 'read', commit } : { kind: 'read' }; @@ -217,7 +220,7 @@ async function originTip(checkout: string, branch: string): Promise<{ kind: 'rea /** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promise<{ files: string[] } | { fault: DeliveryFault }> { if (!OBJECT_ID.test(key.head) || !OBJECT_ID.test(base)) return { fault: { kind: 'unreadable' } }; - const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [base, key.head] }); + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [base, key.head] }, ORIGIN_GIT); if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; const diff = await git(checkout, ['diff', '--no-renames', '--name-only', '-z', base, key.head, '--']); if (diff.code !== 0) return { fault: { kind: 'unreadable' } }; @@ -227,7 +230,7 @@ async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promis /** Whether `commit` holds `ancestor`, read in the run's checkout once it fetched what it lacks. */ async function contains({ runId, checkout }: DeliveryJob, { commit, ancestor }: { commit: string; ancestor: string }): Promise<{ contains: boolean } | { fault: DeliveryFault }> { if (!OBJECT_ID.test(commit) || !OBJECT_ID.test(ancestor)) return { contains: false }; - const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [ancestor, commit] }); + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [ancestor, commit] }, ORIGIN_GIT); if (fetched.missing.includes(commit)) return { fault: { kind: 'unreadable' } }; // A commit the checkout holds brings its history, so an ancestor still missing is not in it. if (fetched.missing.length > 0) return { contains: false }; diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index 07a904dbe..b9fcfe52b 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -229,12 +229,14 @@ async function worktreeHooks(main: string, dir: string, hooks: string): Promise< /** * The commits among `commits` this checkout still lacks after fetching run `run`'s branches from origin. A fetch whose * refspec matches no branch exits 0, so each commit is checked again after it. `fetchError` is git's message when the - * fetch itself failed. + * fetch itself failed. `fetch` bounds the fetch and sets its environment. */ -export async function fetchMissingCommits(dir: string, { run, commits }: { run: string; commits: string[] }): Promise<{ missing: string[]; fetchError?: string }> { +export async function fetchMissingCommits( + dir: string, { run, commits }: { run: string; commits: string[] }, fetch?: { timeoutMs: number; env?: NodeJS.ProcessEnv }, +): Promise<{ missing: string[]; fetchError?: string }> { const lacking = await lackedCommits(dir, commits); if (lacking.length === 0) return { missing: [] }; - const fetched = await git(dir, ['fetch', '--quiet', '--no-tags', 'origin', `+refs/heads/teamai/${run}/*:refs/remotes/origin/teamai/${run}/*`]); + const fetched = await git(dir, ['fetch', '--quiet', '--no-tags', 'origin', `+refs/heads/teamai/${run}/*:refs/remotes/origin/teamai/${run}/*`], fetch); const missing = await lackedCommits(dir, lacking); if (fetched.code === 0 || missing.length === 0) return { missing }; return { missing, fetchError: (fetched.stderr || fetched.stdout).trim() || 'git fetch failed' }; From f16c130aa45777e5195e17717040a5ab95c685d0 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 22:08:46 +0200 Subject: [PATCH 73/77] fix(workflow): pass over a delivery while a writer of its branch is leased nextDelivery picked a delivering step whose branch a leased sibling writes, and deliverPr read the job's writerLeased flag only after the tip checks and the provider lookup had run, to answer busy. So every next made a provider lookup while the sibling ran: with the local driver polling every 2 s, about 1800 gh api calls an hour. nextDelivery now passes over such a step before any I/O. The job no longer carries writerLeased, and deliverPr no longer reads it. The gate in markCreating keeps its check, which covers a lease granted while a delivery runs, and completeDelivery keeps its ahead rule. The tests that leased the sibling before the pick now lease it after the pick, inside the lookup or the git read, so each keeps its race: the four cases of "opens nothing when a sibling ...", and "keeps a step delivering when a report records origin's ahead commit ...". A sibling that has not reported is handed back to the driver, which did not list it as running. "keeps a step delivering while origin's branch is ahead ..." moves the same way, so it still covers the leased half of the ahead rule. A new test covers the gate's check for a sibling leased during the lookup that has not pushed yet. Red: "asks the host nothing while a sibling that writes the shared branch is leased, however many calls come" failed with three gh api lookups where none was expected. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 105 ++++++++++++++++++------ src/workflow/delivery-ledger.ts | 6 +- src/workflow/engine.ts | 3 +- src/workflow/pr-delivery.ts | 10 +-- 5 files changed, 91 insertions(+), 35 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 8cf8db41a..7e3a35e99 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. A reported URL or a request the lookup found is `busy` instead of accepted while the job's `writerLeased`, read under the run lock when the job was picked, says such a step was leased, so the next drain checks that step's commit. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 8e43c35c8..862ff6716 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -371,7 +371,7 @@ describe('delivering a step', () => { const fix = await take(config, runId); if (!fix.workspace) throw new Error('fix has no workspace'); const to = pushWork(root, fix); - const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [], writerLeased: false }; + const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [] }; expect(await deliverPr(job, async () => 'refused')).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); @@ -1155,15 +1155,25 @@ describe('retrying and cancelling a delivery', () => { const gh = fakeGh(path.join(root, 'gh')); const impl = await take(config, runId); await succeed(config, runId, impl, { to: pushWork(root, impl) }); - const lock = deliveryLockPath(config, runId); - expect(await acquireLock(lock)).toBe(true); - const docs = await take(config, runId); - await releaseLock(lock); - const branch = docs.workspace?.branch ?? ''; - const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + const branch = impl.workspace?.branch ?? ''; + let pushed = false; + let docs: Lease | undefined; + let ahead = ''; + const read = worktree.git; + // The sibling is leased and pushes once delivery picked work.impl, before delivery reads the branch. + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + if (!pushed && args[1].includes('ls-remote')) { + pushed = true; + docs = await take(config, runId); + ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + } + return read(...args); + }); - expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); expect(creates(gh)).toEqual([]); + if (!docs) throw new Error('work.docs was never leased'); await succeed(config, runId, docs, { to: ahead }); expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); expect(creates(gh)).toEqual([[branch, 'main']]); @@ -1198,20 +1208,17 @@ describe('retrying and cancelling a delivery', () => { const gh = fakeGh(path.join(root, 'gh')); const impl = await take(config, runId); await succeed(config, runId, impl, { to: pushWork(root, impl) }); - const lock = deliveryLockPath(config, runId); - expect(await acquireLock(lock)).toBe(true); - const docs = await take(config, runId); - await releaseLock(lock); - const branch = docs.workspace?.branch ?? ''; - const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + const branch = impl.workspace?.branch ?? ''; let reported = false; const read = worktree.git; + // The sibling is leased and pushes once delivery picked work.impl, and reports right after delivery read its push. vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + if (reported || !args[1].includes('ls-remote')) return read(...args); + reported = true; + const docs = await take(config, runId); + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); const answer = await read(...args); - if (!reported && args[1].includes('ls-remote')) { - reported = true; - await succeed(config, runId, docs, { to: ahead }); - } + await succeed(config, runId, docs, { to: ahead }); return answer; }); @@ -1386,13 +1393,12 @@ describe('retrying and cancelling a delivery', () => { const { config, root, runId } = await fixture(IMPL_AND_DOCS); const impl = await take(config, runId); await succeed(config, runId, impl, { to: pushWork(root, impl) }); - const lock = deliveryLockPath(config, runId); - expect(await acquireLock(lock)).toBe(true); - const docs = await take(config, runId); - await releaseLock(lock); - const branch = docs.workspace?.branch ?? ''; + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; let sibling: string | undefined; + // Leased only once delivery picked work.impl: a sibling leased before the pick holds the delivery back. const siblingPushes = async () => { + docs = await take(config, runId); sibling = pushOnto(root, branch, { '.github/workflows/ci.yml': 'on: push\n' }); if (recorded) await succeed(config, runId, docs, { to: sibling }); }; @@ -1413,7 +1419,10 @@ describe('retrying and cancelling a delivery', () => { return opened; }); - expect(await leaseNext(config, runId, remote, { ...writable, ...(recorded ? {} : { running: [docs.attempt] }) })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + const answer = await leaseNext(config, runId, remote, writable); + if (!docs) throw new Error('work.docs was never leased'); + // A sibling that has not reported is still leased, and this call did not list it as running, so it gets it back. + expect(answer).toMatchObject({ data: recorded ? { wait: { delivering: ['work.impl'] } } : { lease: { step: 'work.docs', attempt: docs.attempt } } }); const waiting = await state(config, runId); expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); expect(waiting.creating).toBeUndefined(); @@ -1428,6 +1437,54 @@ describe('retrying and cancelling a delivery', () => { expect(create).not.toHaveBeenCalled(); }); + it('opens nothing when a sibling that writes the shared branch is leased while the lookup runs, and checks its commit once it reports', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; + const create = providerDouble(async () => { + docs ??= await take(config, runId); + return undefined; + }, async () => 'https://github.com/fixture/app/pull/1'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + + if (!docs) throw new Error('work.docs was never leased'); + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it('asks the host nothing while a sibling that writes the shared branch is leased, however many calls come', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + + for (let call = 1; call <= 3; call += 1) { + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + } + expect(gh.calls()).toEqual([]); + + await succeed(config, runId, docs, { to: pushOnto(root, branch, { 'docs.txt': 'docs' }) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + }); + it.each([ ['TeamAI would open its pull request', {}, false], ['its driver reported its pull request', { pr_url: 'https://github.com/fixture/app/pull/1' }, true], diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index 3bf46213f..e6ad1dce4 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -24,7 +24,8 @@ export function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep /** * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel * stopped is not delivered. One in `waiting`, whose branch this drain found ahead of the recorded commit, waits for a - * later drain. Undefined when nothing delivers, or when the run settles, which the locked body of `next` finishes. + * later drain, and so does one whose branch a leased step writes, since that step's next push would enter its pull + * request unchecked. Undefined when nothing delivers, or when the run settles, which the locked body of `next` finishes. */ export function nextDelivery( run: RunState, reconcile: (run: RunState) => object, writerLeased: (run: RunState, branch: string) => boolean, waiting: readonly DeliveryKey[], @@ -35,7 +36,7 @@ export function nextDelivery( for (const step of run.steps) { if (step.status !== 'delivering') continue; const key = deliveryKey(step); - if (waiting.some((ahead) => sameKey(ahead, key))) continue; + if (waiting.some((ahead) => sameKey(ahead, key)) || writerLeased(run, key.branch)) continue; const target = deliveryTarget(run, key.branch); if (!run.checkout || !run.base || target === undefined) { deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); @@ -45,7 +46,6 @@ export function nextDelivery( key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, recordedHead: run.copies?.[key.branch]?.commit, childWork: childWorkOff(run, step, key.branch), openedBy: run.steps.find((item) => item !== step && branchName(run.id, item.id, item.pass ?? 1) === key.branch)?.id, - writerLeased: writerLeased(run, key.branch), ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), }; } diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 26f8f522f..d0206d59c 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -994,7 +994,8 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv }; const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => { // Steps still delivering here are being delivered by another process of this machine, waited for a busy run - // lock, or wait for a report of the commit their branch is ahead at on origin; a driver polls until they settle. + // lock, or wait for a report: of the commit their branch is ahead at on origin, or of a leased step that writes + // their branch. A driver polls until they settle. const delivering = run.steps.filter((step) => step.status === 'delivering').map((step) => step.id); const reason = open ? `${open.id} is leased` : delivering.length > 0 ? `${delivering[0]} is delivering` : 'canTakeLease is false'; return answer({ wait: { reason, ...(delivering.length > 0 ? { delivering } : {}) } }); diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts index b1df87d79..d259bb9a2 100644 --- a/src/workflow/pr-delivery.ts +++ b/src/workflow/pr-delivery.ts @@ -27,8 +27,6 @@ export interface DeliveryJob { childWork: Array<{ step: string; branch: string; commit: string }>; /** The workflow step whose copy the branch is, when another step's: retrying it runs its steps again in a new branch. */ openedBy?: string; - /** A step that writes the branch was leased, so its next push would enter a pull request unchecked: none is taken yet. */ - writerLeased: boolean; /** The branch the pull request goes into. */ target: string; /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ @@ -83,8 +81,8 @@ const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }) * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch * is looked up, and only when there is none, origin's branch is still at the checked commit, and `gate` opens, is one - * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, a - * step that writes it was leased, or the gate could not answer, and the job is for a later call, which checks it again. + * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, or + * the gate could not answer, and the job is for a later call, which checks it again. * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. */ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { @@ -117,7 +115,7 @@ export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise Date: Sat, 24 Oct 2026 22:34:22 +0200 Subject: [PATCH 74/77] test(workflow): pin that a timeout stops delivery before anything opens Timeouts and budgets now settle in reconcileRun, which both the delivery pick and the create gate apply. A deadline that passes while a step delivers leaves no lookup and no create; one that passes during the lookup leaves no create. The dev doc states where limits act. Red: with the gate ignoring a settled outcome, 'opens nothing once a timeout passes while its delivery looks for an open pull request' failed with createPullRequest called once; with the pick not reconciling, 'opens nothing once a timeout passes while its step delivers' failed with the lookup called once. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 47 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 00fe65745..2c372ccf8 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. Staging runs inside the attempt at every `max_parallel` (`stageThenStart`), so the driver keeps polling while it stages, and a cancel aborts staging. A step whose attempt was cancelled, or whose lease expired, while it staged never starts. `stageLease` calls `runSetup` (`setup.ts`) last, only in a workspace it just created, with setup's own keys, and removes the worktree when setup fails. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. A condition that stops a run belongs in `reconcileRun`, which this gate applies, so no pull request opens once it holds. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index a1192b42f..326f6eb26 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1777,3 +1777,50 @@ describe('retrying and cancelling a delivery', () => { expect(run.creating).toBeUndefined(); }); }); + +describe('limits that stop a delivering run', () => { + const at = (minutes: number): Date => new Date(Date.UTC(2026, 0, 1) + minutes * 60_000); + const TIMED_WORK = ` - id: work + workspace: new + timeout: 10m + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: after + needs: [work] + command: echo after +`; + + it.each([ + ['while its step delivers', 'before'], + ['while its delivery looks for an open pull request', 'lookup'], + ] as const)('opens nothing once a timeout passes %s', async (_case, when) => { + vi.useFakeTimers({ toFake: ['Date'] }); + try { + vi.setSystemTime(at(0)); + const { config, root, runId } = await fixture(TIMED_WORK); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lookup = vi.fn(async () => { + if (when === 'lookup') vi.setSystemTime(at(15)); + return undefined; + }); + const create = providerDouble(lookup, async () => 'https://github.com/fixture/app/pull/1'); + if (when === 'before') vi.setSystemTime(at(15)); + + await leaseNext(config, runId, remote, writable); + const run = await state(config, runId); + expect(create).not.toHaveBeenCalled(); + expect(run.deliveries).toBeUndefined(); + expect(run.creating).toBeUndefined(); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'timeout' } }); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'cancelled' }); + if (when === 'before') expect(lookup).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); From ab44ccf0c9eb17659899aca38dd75cc9031e5b48 Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 23:01:41 +0200 Subject: [PATCH 75/77] fix(workflow): fail a workflow step that would deliver nothing and lose child work A workflow step with deliver: pr whose own branch holds nothing beyond the run base delivers nothing, so the child-work check never ran. A step inside it that wrote a branch of its own then succeeded with its work in no pull request. The step now fails with the child_work_left_out message when such work exists. Red: 'fails a workflow step whose own branch holds nothing beyond the run base while a step inside it wrote another branch' ended the run succeeded. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 17 ++++++++++++++++ src/workflow/engine.ts | 15 ++++++++++---- src/workflow/workspace.ts | 26 +++++++++++++++++++++---- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 2c372ccf8..ffc53f694 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. Staging runs inside the attempt at every `max_parallel` (`stageThenStart`), so the driver keeps polling while it stages, and a cancel aborts staging. A step whose attempt was cancelled, or whose lease expired, while it staged never starts. `stageLease` calls `runSetup` (`setup.ts`) last, only in a workspace it just created, with setup's own keys, and removes the worktree when setup fails. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. A workflow step whose branch holds nothing beyond the run base opens no pull request, so `finishStep` fails it with the same message when such work exists (`workLeftUndelivered`). It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 326f6eb26..153f1dc3d 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -677,6 +677,23 @@ describe('where a pull request goes', () => { expect(run.deliveries).toBeUndefined(); }); + it('fails a workflow step whose own branch holds nothing beyond the run base while a step inside it wrote another branch', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: work\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: a\n workspace: continue\n command: node -e ""\n`, + ` - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const [work, b] = [`teamai/${runId}/work/p1`, `teamai/${runId}/work-b/p1`]; + expect(run.steps.find((step) => step.id === 'work.a')).toMatchObject({ status: 'succeeded', changes: { branch: work, to: run.base?.commit } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`step "work.b" wrote branch ${b}, which branch ${work} does not hold`) } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + }); + it('opens the pull request of a workflow step once a later step inside it merges the branch another step wrote', async () => { const { config, cwd, root, runId } = await fixture(`${COPY_AND_SIDE_BRANCH} - id: c\n needs: [b]\n workspace: continue\n command: ${bump('c.txt')}\n`); const gh = fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 912369d5b..d6ebb6de0 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -25,9 +25,9 @@ import { budgetSpent, countLateTokens, expire, itemsSpent, leaseDeadline, ownDea import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, isUnder, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspacePlan } from './workspace.js'; +import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, workLeftUndelivered, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspacePlan } from './workspace.js'; import { - deliverPr, findDelivered, reportedPrMismatch, type DeliveryJob, + deliverPr, deliveryFailure, findDelivered, reportedPrMismatch, type DeliveryJob, } from './pr-delivery.js'; import { sanitizePrUrl, type PrUrl } from './pr-url.js'; import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate } from './delivery-ledger.js'; @@ -565,9 +565,16 @@ function finishStep(run: RunState, state: StepState, definition: WorkflowStep, r fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); return; } - const delivery = definition.deliver === 'pr' && (!definition.forEach || isItem(state)) ? deliverableOf(run, state, definition) : undefined; + const delivers = definition.deliver === 'pr' && (!definition.forEach || isItem(state)); + const delivery = delivers ? deliverableOf(run, state, definition) : undefined; + // A branch that holds nothing beyond the run base opens no pull request, so work a step inside wrote elsewhere would be lost. + const left = delivers && !delivery && definition.workflow !== undefined ? workLeftUndelivered(run, state) : undefined; if (delivery) startDelivery(state, settlement, delivery); - else succeed(state, settlement); + else if (left) { + const key = { step: state.id, pass: state.pass ?? 1, settledAt: result.settledAt, branch: left.branch, head: left.head }; + const failure = deliveryFailure({ runId: run.id, key }, { kind: 'child_work_left_out', ...left.child }); + fail(state, { ...settlement, message: failure.message, failure }); + } else succeed(state, settlement); } function childOrUnreadable(run: RunState, scope: Scope, states: StepState[], state: StepState, definition: WorkflowStep): Scope | { unreadable: string } { diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index d666df660..3f9102acc 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -172,15 +172,33 @@ export function deliverableOf(run: RunState, state: StepState, step: WorkflowSte let branch = changes?.branch; let head = changes?.to; if (step.workflow !== undefined) { - const own = branchName(run.id, state.id, state.pass ?? 1); - const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status !== 'cancelled' && changesOf(item)) - .sort((left, right) => ('settledAt' in right ? right.settledAt : 0) - ('settledAt' in left ? left.settledAt : 0))[0]; - branch = run.copies?.[own]?.commit !== undefined ? own : changesOf(latest)?.branch; + branch = workflowBranch(run, state); head = branch === undefined ? undefined : run.copies?.[branch]?.commit; } return branch === undefined || head === undefined || head === run.base?.commit ? undefined : { branch, head }; } +/** The branch workflow step `state` delivers: its own copy, else the branch its latest step wrote. */ +function workflowBranch(run: RunState, state: StepState): string | undefined { + const own = branchName(run.id, state.id, state.pass ?? 1); + if (run.copies?.[own]?.commit !== undefined) return own; + const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status !== 'cancelled' && changesOf(item)) + .sort((left, right) => ('settledAt' in right ? right.settledAt : 0) - ('settledAt' in left ? left.settledAt : 0))[0]; + return changesOf(latest)?.branch; +} + +/** + * Work a step inside workflow step `state` wrote beyond the run base, when `state` delivers nothing of its own: no + * pull request would hold it. Undefined when there is none. + */ +export function workLeftUndelivered(run: RunState, state: StepState): { branch: string; head: string; child: { step: string; branch: string } } | undefined { + const branch = workflowBranch(run, state); + const base = run.base?.commit; + if (branch === undefined || base === undefined) return undefined; + const child = childWorkOff(run, state, branch).find((item) => item.commit !== base); + return child && { branch, head: base, child }; +} + /** * The work steps inside `state` that succeeded left on other branches than `branch`, the one `state` delivers, which no * step inside it delivers: a pull request of `branch` leaves out any of it that `branch` does not hold. From a8f6b6bd8b4681214622c304049f46c1824a6e4d Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 23:01:49 +0200 Subject: [PATCH 76/77] fix(workflow): take no found pull request while its branch is unchecked The create gate answers busy while a writer of the branch is leased, but a pull request the lookup found, or a driver reported, skipped the gate and settled the step, releasing its dependents before the writer reported. completeDelivery now applies the conditions it already used for an ahead tip: while a writer is leased, or a report recorded another commit since the pick, the step stays delivering and a request the job did not open gets no receipt. One it opened still gets its receipt. Red: 'takes no open pull request when a sibling that writes the shared branch is leased while the lookup finds it' found the step succeeded. --- docs/dev/workflow-engine.md | 2 +- src/__tests__/workflow-delivery.test.ts | 26 +++++++++++++++++++++++++ src/workflow/delivery-ledger.ts | 17 ++++++++++------ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index ffc53f694..afdef683b 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -13,7 +13,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. Staging runs inside the attempt at every `max_parallel` (`stageThenStart`), so the driver keeps polling while it stages, and a cancel aborts staging. A step whose attempt was cancelled, or whose lease expired, while it staged never starts. `stageLease` calls `runSetup` (`setup.ts`) last, only in a workspace it just created, with setup's own keys, and removes the worktree when setup fails. - **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. -- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. A workflow step whose branch holds nothing beyond the run base opens no pull request, so `finishStep` fails it with the same message when such work exists (`workLeftUndelivered`). It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The same two conditions hold back a pull request: the step stays `delivering`, and one the job found or was told of gets no receipt, while one it opened does. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. A workflow step whose branch holds nothing beyond the run base opens no pull request, so `finishStep` fails it with the same message when such work exists (`workLeftUndelivered`). It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. - **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. - **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts index 153f1dc3d..e3f1c37ee 100644 --- a/src/__tests__/workflow-delivery.test.ts +++ b/src/__tests__/workflow-delivery.test.ts @@ -1482,6 +1482,32 @@ describe('retrying and cancelling a delivery', () => { expect(create).not.toHaveBeenCalled(); }); + it('takes no open pull request when a sibling that writes the shared branch is leased while the lookup finds it, and checks its commit once it reports', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; + const create = providerDouble(async () => { + docs ??= await take(config, runId); + return 'https://github.com/fixture/app/pull/1'; + }, async () => 'https://github.com/fixture/app/pull/2'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.deliveries).toBeUndefined(); + + if (!docs) throw new Error('work.docs was never leased'); + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + it('asks the host nothing while a sibling that writes the shared branch is leased, however many calls come', async () => { const { config, root, runId } = await fixture(IMPL_AND_DOCS); const gh = fakeGh(path.join(root, 'gh')); diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts index e6ad1dce4..ded06261c 100644 --- a/src/workflow/delivery-ledger.ts +++ b/src/workflow/delivery-ledger.ts @@ -83,20 +83,25 @@ function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrU * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. Origin's * branch ahead of the recorded commit fails it only once no report of that tip can come: no step that writes the branch - * is leased, and no report recorded another commit on it since the job was picked. + * is leased, and no report recorded another commit on it since the job was picked. Until then a pull request settles + * nothing either, and one the job did not open gets no receipt: a later call checks the branch again and finds it. * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. */ export function completeDelivery( run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, reconcile: (run: RunState) => unknown, writerLeased: (run: RunState, branch: string) => boolean, ): { requeueEnd: boolean } { - const receipted = outcome.ok && recordReceipt(run, job.key, job.target, outcome.pr); - if (outcome.ok) forgetCreate(run, { ...job.key, target: job.target }); + const create = { ...job.key, target: job.target }; + const unchecked = (): boolean => writerLeased(run, job.key.branch) || run.copies?.[job.key.branch]?.commit !== job.recordedHead; + const settles = !unchecked(); + // A request this job opened gets its receipt whatever became of the branch; one it found or was told of waits. + const taken = settles || run.creating?.some((listed) => sameCreate(listed, create)) === true; + const receipted = outcome.ok && taken && recordReceipt(run, job.key, job.target, outcome.pr); + if (outcome.ok && taken) forgetCreate(run, create); if (run.status === 'running') reconcile(run); const step = deliveringStep(run, job.key); - const awaited = !outcome.ok && outcome.fault.kind === 'branch_ahead' - && (writerLeased(run, job.key.branch) || run.copies?.[job.key.branch]?.commit !== job.recordedHead); - if (step && outcome.ok) delivered(step); + const awaited = !outcome.ok && outcome.fault.kind === 'branch_ahead' && unchecked(); + if (step && outcome.ok && settles) delivered(step); else if (step && !outcome.ok && !awaited) deliveryFailed(step, deliveryFailure(job, outcome.fault)); return { requeueEnd: receipted && run.status !== 'running' }; } From ff5de032f14e01e6e76fc841a01ecb045bd5ab2e Mon Sep 17 00:00:00 2001 From: Saul Moro Date: Sat, 24 Oct 2026 23:02:35 +0200 Subject: [PATCH 77/77] refactor(workflow): name the fault fields of left-out child work --- src/workflow/engine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index d6ebb6de0..431e051b9 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -572,7 +572,7 @@ function finishStep(run: RunState, state: StepState, definition: WorkflowStep, r if (delivery) startDelivery(state, settlement, delivery); else if (left) { const key = { step: state.id, pass: state.pass ?? 1, settledAt: result.settledAt, branch: left.branch, head: left.head }; - const failure = deliveryFailure({ runId: run.id, key }, { kind: 'child_work_left_out', ...left.child }); + const failure = deliveryFailure({ runId: run.id, key }, { kind: 'child_work_left_out', step: left.child.step, branch: left.child.branch }); fail(state, { ...settlement, message: failure.message, failure }); } else succeed(state, settlement); }