diff --git a/docs/dev/workflow-engine.md b/docs/dev/workflow-engine.md index 86890de4f..afdef683b 100644 --- a/docs/dev/workflow-engine.md +++ b/docs/dev/workflow-engine.md @@ -9,12 +9,14 @@ Each rule below spans more than one file. The rule names the code that owns it, - **A run ends only in `settleRun`.** The shared `finishRun` path calls it when settling reaches the top workflow's result, before it hands any lease back or out, and on a call that takes no lease too. `reportAttempt` first reconciles earlier reports, so an end cancels a later report before it can advance the revision. It records an accepted outcome and never ends the run. After a failed report, the next `next` skips the dependents and ends the run. Respond, cancel and retry in `run-controls.ts`, and next in `engine.ts`, share that path. Whole-run cancel settles directly as `cancelled`, without reconciling another round. To run code when a run ends, hook into `finishRun`. Do network I/O only after `withRun` releases the run lock. That path queues the end record, a local write, under the lock, and saves whether it did in the run's `endRecordQueued` field. When `updateRun` then fails to save the run file, the queue is one save ahead of the run file, and the next `next` settles the run again and overwrites the queued `.end.json`. A later `next` reports a record state only for a run whose field is true (`runRecordState` in `records.ts`). - **Reconcile the latest report before applying another or handing out a lease.** `reconcileRun`, shared by `leaseNext` and `reportAttempt`, settles every driverless transition to a fixpoint at the run's current revision, through `settleScope` and `settleItems`, including workflow starts, fan-out expansion, render failures and bound exhaustion. `reportAttempt` advances the revision only after it, so only the latest revision can be unreconciled. A reached end applies before a loop decision. Only after reconciliation does `leaseNext` hand back an omitted open lease or issue the first ready leaf in declaration order that this driver can take. A running workflow occupies its parent's position in that order. Settling returns every ready leaf in that order (`ready`), and `leaseNext` walks it. A ready item does not count against `max_parallel` until it is leased, and one `next` leases at most one step, so a held-back item leaves its room to the next one without exceeding the cap. `canTakeLease: false` still reconciles driverless transitions. Every changed run is saved, including a `wait`. - **A report is checked against the revision of its own lease.** `reportAttempt` and `retryStep` increment `revision`; skips, `if`, `end`, and the creation of child steps inside `next` never do. A lease keeps the run revision it was handed out at, and `reportAttempt` compares a report with that, so the report of a parallel lease does not make it stale. An attempt that is not an open lease keeps the older answers: `stale_revision` when its revision is not the run's, otherwise `wrong_attempt`. -- **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. +- **An end or break stops what is still open under its workflow.** `stopWhere` cancels every leased step under the workflow that reached the end, its sub-workflows included, and keeps the attempt on the step and in `cancelledAttempts`. It cancels running workflow steps there and skips pending and waiting ones. The walk takes a path predicate and failure. End and break construct their own end_reached failure; manual cancel and retry supply theirs. Step descendants use the same `isUnder` predicate as `archiveUnder` in `loop.ts`. A cancelled delivering step keeps its agent settlement and its original `settledAt` for explicit recovery, records and scoring. Automatic delivery and workflow-item branch summaries exclude cancelled children. Cancel records the ids this walk changed before reconciliation can settle or skip other steps. `reportAttempt` answers a cancelled attempt `lease_cancelled` before it checks the driver; `cancelledAttempts` is never trimmed. Every `next` answer lists in `cancelled` the attempts the driver says it runs that are no longer its open leases, so the driver learns of a cancellation from any answer, `done` included. - **The local driver overlaps steps that do not share a checkout.** `runLocalDriver` runs up to `workflows.max_parallel` steps (local config, default 1) and sends the ones it has started as `running`. A command with no workspace, and a prompt in the current checkout, share that checkout, so only one of them runs there at a time. A prompt earlier in the queue for that checkout blocks a later command there. The driver starts the prompt when the checkout is free and fewer than `max_parallel` steps are in flight; commands in other worktrees keep running. A prompt with its own worktree does not take the checkout. A headless prompt takes one of the slots, and the driver keeps leasing steps for the others while it runs. An interactive prompt uses the terminal, starts only when no command is running, and no step is leased while it runs. On `cancelled`, it drops a held lease, and it stops a running command with SIGTERM, then SIGKILL after 5 s while any captured descendant or process in a detached group remains, even after its command exited. Attached commands snapshot their descendant tree and start times before stopping, then probe and send SIGKILL only to descendants whose start times still match. If the identity snapshot fails, only the shell is stopped; detached commands use their process group. It does not send a stopping command as `running` again, so `next` does not cancel it twice. Dying commands still take a slot, and a new attempt of their step waits until they are gone. The signal policy follows `max_parallel`. At 1, a command runs in teamai's process group, as `env exec` runs one (`src/env-exec.ts`): the terminal's Ctrl-C reaches it, and it can read `/dev/tty`. A signal `env exec` would pass on (SIGTERM or SIGHUP, and SIGINT or SIGQUIT when the terminal does not reach the command) makes the driver stop the command, wait for it, and exit with that signal. Above 1, each command runs detached in a process group of its own, with no terminal, and SIGINT, SIGQUIT, SIGTERM or SIGHUP makes the driver stop every group, wait for them, and exit with that signal. Windows uses `taskkill /T /F` in place of the group kill. Staging runs inside the attempt at every `max_parallel` (`stageThenStart`), so the driver keeps polling while it stages, and a cancel aborts staging. A step whose attempt was cancelled, or whose lease expired, while it staged never starts. `stageLease` calls `runSetup` (`setup.ts`) last, only in a workspace it just created, with setup's own keys, and removes the worktree when setup fails. -- **A workspace is a git contract on the lease, and the driver is the only code that touches git.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. +- **A workspace is a git contract on the lease. The driver owns workspace git; TeamAI reads the delivered commit and opens the PR outside the run lock.** `planWorkspace` in `workspace.ts` chooses the mode. An explicit `workspace` wins. A command that omits it is read-only, even inside a copy; when it exits 0 and changed the checkout, the driver fails it `read_only_dirty`. That includes a commit that leaves the worktree clean, a switch to another branch or a detach at the same commit, a change that only stages an existing edit, another staged hunk that leaves the status mixed, a permission change on a file that was already dirty, an edit to a quoted non-ASCII path, a symlink whose target changes while the file it names stays the same, and an edit to the destination of a staged rename, and a change inside a submodule that was already dirty. An interactive prompt that sets `workspace: read-only` is checked the same way. An interactive prompt that does not may change the checkout, and those edits stay. Conflict paths, including agent files, are read without Git's quoting, so a non-ASCII name is still checked for leftover markers. A prompt that omits it continues the nearest enclosing `new` or `continue` workflow, and a `for_each` ancestor makes that `new`, so items do not share a worktree. `leaseNext` puts the contract on the lease for `new` and `continue`, and for a read-only step inside a copy (`mode: 'read-only'`, `merge: []`). That reader's branch is the one its declared `needs` wrote or read (`changes` or `read`, through items and children, but not `after` items or inherited needs), or the enclosing copy's when they touched none; needs on several branches fail `workspace_base`. It needs `continue` in `can.workspace` and the copy's owner, as `continue` does. A read-only step outside any copy gets no contract and runs in the active checkout. The local driver stages a reader without merges, takes its snapshot over the base overlay, and afterwards restores the overlay; when the step changed the copy or left its branch, it keeps the state on `refs/teamai/kept/` (`git add -A`, then `write-tree` and `commit-tree` on HEAD; not the stash list, which every worktree shares), and pushes each submodule commit that tree records and the read commit does not into the checkout's git dir as `refs/teamai/kept-modules//` (at any depth, without the user's push hooks or `push.recurseSubmodules`), since a gitlink does not keep its commit reachable once the copy is removed, then runs `git checkout -f `, `git reset --hard` to the commit it read, `git clean -fd`, and `git submodule update --checkout --recursive` on the submodules it changed, so a later writer never commits a reader's edits. It resets nothing when a repository among the changed paths has uncommitted or untracked files, or a nested submodule commit its parent does not record, which the keep cannot hold. The copy counts as put back only when HEAD is `refs/heads/` at that commit and `git status --porcelain --untracked-files=all --ignore-submodules=none` prints the same lines it printed when the reader was staged; ignored files are left in place. Otherwise the reader fails `workspace_base`. `stageLease` writes a `teamai-unrestored` marker, with that status, in the worktree's git dir before the reader's overlay, and the driver removes it once the copy is put back or left untouched, so a driver killed mid-reader still leaves it. `stageLease` refuses later leases in a marked copy until the copy is back, when it removes the marker. A status line does not show content, so a marker whose status was not empty stays until the person who puts the copy back deletes it. `ensureWorktree` also refuses an existing worktree whose HEAD is not on its branch or a directory that is not a worktree, and prunes and recreates one whose directory was deleted. The reader reports `read` (`branch`, `base`, `commit`) and never `changes`. `report` returns `invalid_report` for `read` on any other lease, and for `changes` or a success without `read` on a reader's lease. `run.copies[].commit` and fan-out `branches` follow `changes` only. `cleanup` comes from the lease's contract, so a reader that was the copy's last user removes it too. `leaseNext` holds a step back for this driver, and tries the next ready leaf, when the driver's `can.workspace` lacks that mode, or when a `continue` copy belongs to another driver (`run.copies`). It answers `wait`, with every held step's reason joined by `; `, only when every ready leaf is held back. Driver capabilities stay out of `reconcileRun`, which `reportAttempt` and the run controls also call. It fails `workspace_base` when `continue` has no enclosing `new`, or when the run has no base. `workflow start` in a git checkout records the base (commit, and the branch unless HEAD is detached); otherwise the first `next` that sends `base` sets it, and a later `base` never changes it. The local driver fails a lease whose `workspace` it cannot read with `workspace_base` instead of running it in the active checkout. The branch is `teamai///p` (`branchName`); a `[n]` in the path becomes `-n`. The first branch in `needs` order is where the step starts, and the rest are merged in that order. An item's `after` branches come first. The local driver (`local-driver.ts`, `worktree.ts`) adds the worktree under `/.teamai/workflow-worktrees/`, points `core.hooksPath` at an empty directory, and replaces `AGENT_PATHS` (`AGENTS.md`, `CLAUDE.md`, `.agents`, `.githooks`, and each default `toolPaths` entry's project directory, project MCP file and Copilot `.github` paths) with the run's base commit. An agent path under a symlinked parent fails `base_config` before anything is copied or removed, since both would follow the link. It then gives the worktree the inline `pull` a new worktree gets (`prepareCheckout`), which delivers the team's skills, rules, agents and MCP and records that checkout's team revision. The overlay is taken after that delivery, so those files are not the step's commit. A path the step edits is committed. A step that left HEAD off the leased branch fails `workspace_base` before anything is committed, reported as `changes`, or pushed, and the worktree stays as it left it. A path it leaves untouched is put back from the branch. A merge conflict stops the merge: a command fails `merge_conflict`, and a prompt is told the branches that remain. Snapshots and commits read submodules with `--ignore-submodules=none`, and after `git add -A`, which skips a submodule whose `ignore` is `all`, each submodule is staged by name, so `submodule..ignore` and `diff.ignoreSubmodules` hide no change from a reader's check, its keep, or a writer's commit. The driver commits as `TeamAI ` and reports `changes` (`branch`, `base`, `from`, `to`, `commits`). It pushes when `origin` exists and this step made commits, whether the step succeeded or failed, and also when a later attempt adds no commit but this branch has commits `origin` does not have, including a first push that never landed and an origin with no tracking refs yet. A failed push fails a step that succeeded with `push_failed`, and the worktree stays. `report` sets `cleanup` only when the step succeeded and no step that can still run uses that branch, judged on the run as the next `next` settles it: steps a loop runs again count, and a workflow or `for_each` step that has not started counts by the branch `plannedBranch` gives, the one around it unless it opens its own; only then does the driver run `git worktree remove --force`. It does not delete the branch, and it never resets, cleans, or switches the active checkout. A command that starts its own sandbox, such as `docker`, stays read-only. What it writes does not feed `new`, `branches`, or a later step. `session: continue` and `session.from` are step reads. The engine puts `{ tool, id }` on the lease, with `earlier`, the sessions that one continued, newest first, and never copies a session. The driver resumes Claude and Codex by that id, copies another tool through `AgentSession.copy`, restores a copy from the team session archive when neither works, and tries each `earlier` session the same way before it starts fresh. OpenCode has no resume flag, so it starts fresh. A driver that continued the lease's session or an earlier one, by that id or in a copy, echoes the one it continued as `resumed` in its report. One that started a new session leaves `resumed` out. `reportAttempt` answers `invalid_report` for a `resumed` on a step that resumes no session, and for one that no pass of the run reported and no link continued (`knowsSession`). When the reported `tool` and `session` differ from `resumed`, `reportAttempt` appends `{ from: resumed, to }` to `run.sessionContinuations`. `sessionOf` follows those links from the session a step would resume, taking the latest link from each session, so a later step resumes the copy that holds the last turns, whichever machine or tool made it. The snapshot publishes the links, so `resumableSessions` on this machine and the claim on the next one both see them. A step that resumed by the same id adds no link, so a run on one machine with one tool leases the same sessions as without links. - **A driver pushes the commits it reports, and fetches the commits it is leased.** Every driver keeps these two git rules, so a run another driver or another machine continues finds each commit on `origin`. Whenever `origin` exists, a driver pushes the commits it reports in `changes` before it reports them, failed steps included; the local driver's `finishWorkspace` pushes for every outcome. A push that fails still reports the commits. A step that succeeded fails `push_failed`. A step that failed keeps its own failure, and the push sentence is added to its message: a retry must fix the step first, and a `push_failed` in its place would hide that cause from the person and from the run record. Before it starts from, merges, or applies a lease commit (`start`, each `merge[]` entry, `configCommit`), a driver checks it with `git cat-file -e ^{commit}`. For a missing one it fetches `+refs/heads/teamai//*:refs/remotes/origin/teamai//*`, and checks again, since a refspec fetch that matches no branch exits 0. A fetch by commit id is not used: a server can refuse it, and a fetched commit no ref holds can be collected later. A commit still missing fails `workspace_base` naming the commit and its branch, before anything merges or runs, so it is never a file-less `merge_conflict` or a prompt told to resolve one. The local driver does this in `stageLease` through `fetchMissingCommits`. `workflowGitConformanceScenarios` in `conformance.ts` checks both rules against a bare origin, and `workflow-git-conformance.test.ts` runs them against the local driver. +- **`next` delivers pull requests, outside the run lock.** A `deliver: pr` step whose agent succeeded becomes `delivering` (`startDelivery`), holding `delivery` (`branch`, `head`, and a kept `reported` URL). It is neither `stopped()` nor `settled()`, so dependents wait, and `startedItems` does not count it, since `max_parallel` bounds agents. Every `leaseNext` calls `drainDeliveries` after an optional claim and before its locked body, whichever driver called it. The drain takes `deliveryLockPath` once; while another process of this machine holds it, the call delivers nothing and `next` answers `wait.delivering`, which the local driver polls. Each round picks a job under the run lock (`nextDelivery`, then `unresolvedCreate` in `delivery-ledger.ts`), runs `deliverPr` or `findDelivered` in `pr-delivery.ts` with no lock held, and applies the outcome under the run lock with `completeDelivery` or `resolveCreate`, which read the immutable job rather than the step. `deliverPr` requires origin's tip of the branch (`ls-remote`) to be the reported head or the latest `copies[branch].commit` a report recorded, and a recorded later tip to contain the reported head (`merge-base --is-ancestor`), else `branch_rewritten`. A tip past the recorded commit that still contains it is `branch_ahead`. `completeDelivery` keeps the step `delivering` while a step that writes the branch is leased (`branchWriterLeased` in `engine.ts`), since that step may push before it reports, or when `copies[branch].commit` is no longer the job's `recordedHead`, since a report recorded a commit while the drain read the branch. The drain does not pick it again in that call. Otherwise no report of that tip can come, and the step fails at once with the `branch_ahead` message, which says the tip holds the checked changes and later ones no report recorded. The same two conditions hold back a pull request: the step stays `delivering`, and one the job found or was told of gets no receipt, while one it opened does. The `branch_moved` and `branch_missing` remedies name the recorded commit. `deliverPr` fetches and diffs the selected tip against the run base on the run's checkout, running every git call that reaches origin within `DELIVERY_TIMEOUT_MS` and with `NON_INTERACTIVE_GIT_ENV`, and rejects protected paths (`protectedPaths` in `worktree.ts`: `AGENT_PATHS` plus `DELIVERY_ONLY_PATHS`) before it accepts a reported URL, looks up or creates. When a recorded later tip changes protected paths, it also diffs the step's own head: a clean own head is `later_protected_path`, a `delivery_failed` that names the later commit, so a retry delivers again instead of running the agent. Its remedies are the ones the run accepts: put origin back at the step's own head, or retry the workflow step whose copy the branch is (the job's `openedBy`). The job of a workflow step carries `childWork` (`childWorkOff` in `workspace.ts`): the head of each step inside it that succeeded on another branch that no step inside it delivers. Each must be an ancestor of the tip, else `child_work_left_out`, since the pull request would leave that work out. A workflow step whose branch holds nothing beyond the run base opens no pull request, so `finishStep` fails it with the same message when such work exists (`workLeftUndelivered`). It looks the branch up before the create and again after it, and reads origin's tip again between that first lookup and the gate. Only a lookup's URL is ever recorded. A post-create lookup that fails, or that finds none after a create that answered, fails delivery with `create_unconfirmed` and keeps the unresolved entry for a later lookup. It treats a found URL on another host than the origin's pull requests as a fault. Its gate, `markCreating`, takes the run lock right before the create: it reconciles, refuses unless the same pass of the same commit still delivers, answers `busy` when `copies[branch].commit` is no longer the job's `recordedHead` or while a step that writes the branch is leased (`branchWriterLeased`), and lists the request in `run.creating`. `nextDelivery` passes over a step while a step that writes its branch is leased (`branchWriterLeased`), so no git or provider call runs for it, and no reported URL or found request is accepted, until that step's commit can be checked. The gate's check covers a lease granted while the delivery runs. Timeouts (`expire` in `limits.ts`) and budgets settle in `reconcileRun`, which both `nextDelivery` and this gate apply, so no delivery is looked up, opened or accepted once a limit stops the step or the run. A new condition that stops a run belongs there too. A run lock it cannot take is `busy` too, as is a tip that moved during the lookup, and neither is a refusal: nothing is listed or opened, the step stays `delivering`, and the drain stops for that call, so the next one checks the branch again. A new receipt on a run that already ended queues the end record under the run lock, and it publishes after the drain releases the delivery lock. In `reportAttempt`, the lookup that vets a reported `changes.pr_url` (`reportedPrMismatch`) is the only delivery I/O, and it runs before the run lock. On the origin's host, a lookup that fails or answers an unsafe URL drops the reported URL. No provider code merges; `no-provider-merge.test.ts` scans `src/providers/**`. +- **A pull request is reused by provider lookup, never by a stored receipt.** `delivery-schema.ts` owns the schemas of delivery receipts and unresolved creates, and `CommitIdSchema`. It imports only `zod` and `pr-url.ts`, so `snapshot.ts` and the run store load no delivery code or provider, which `no-provider-merge.test.ts` checks. `delivery-ledger.ts` owns their pure updates. The engine passes `reconcileRun` into the ledger and keeps the drain and create gate. Three fields keep what delivery did. `run.deliveries` is an append-only receipt list, deduplicated by URL, that only `recordReceipt` writes, for `completeDelivery` and `resolveCreate`. `deliveryTarget` in `workspace.ts` reads which branches delivered, `buildEndRecord` in `records.ts` reads it as `prs` in list order, and the snapshot publishes it. `copies[].from` is the branch a copy grew from: `rememberCopy` writes it once, at the copy's first lease, from the contract `planWorkspace` ordered (`after`, then `needs`, then inherited needs), before the copy has a commit. `deliveryTarget` walks it to the nearest branch with a receipt, else `run.base.ref`, and the snapshot publishes it and refuses one outside `teamai//`. `run.creating` lists each request TeamAI asked a host to open and holds no receipt for. `markCreating` appends it, `completeDelivery` removes it with the receipt of a delivered outcome and keeps it on a failed one, and `resolveCreate` applies a lookup of it: found appends the receipt and removes it, unsafe removes it immediately, and none or failed removes it after its second none or third failed answer. Each entry keeps total, none and failed lookup counts in `lookups`, set to zero by the gate each time it opens, including for a request listed before, and advanced only by `resolveCreate`. `unresolvedCreate` picks each at most once per drain, never in the drain that created or failed it, after the delivering steps, in an ended run too, and `findDelivered` reads only the origin and `findOpen`. It touches no step, so a retry, cancel or archive cannot lose the request, and a lookup cannot settle or fail a step. A delivering step whose request is listed still looks up before it creates again. The snapshot publishes the list, so a claimer looks the requests up. - **A run changes machines only through `handoff.ts`.** `publishRun` and the claim behind `claimRun` and `resumeRun` are the only writers of `workflows/runs//state.json` on `teamai-reports`, and they write it inside `updateReports`. A publish writes only while origin shows this install (`installOwner`, from `/workflows/owner.json`) and the state revision this machine last saw (`acknowledged`). An unchanged body writes nothing. A publish checks the business origin without a lock, then answers `state_moved` when, under the locks, the run's base, a copy's commit, or the sessions it archived differ from what it checked (`checkedOutsideLocks`). Anything else that moved meanwhile is published as the locks find it, so a run a driver is still advancing can be published. `resumeRun` continues a run locally when origin has no snapshot and this machine never saw one acknowledged (`stateRevision` 0), since no other install can have claimed it. A claim reads origin under the reports lock and releases it, decides outside the lock (`definitionsOf` can fetch and sync a shared clone, and `checkOrigin` runs `ls-remote`, only for a running run: a settled one leases nothing until a retry reopens it, and the local driver's `reconcileTakenCopies` waits for that too), then writes only while origin still holds the `write` id it read (`replaceSnapshot`). The compare key is the `write` id, not the revision, because two claims of one revision both write `stateRevision + 1`. A moved or missing snapshot answers `state_moved`. Every `next` that answers `done`, and every run control, calls `publishRunEnd`, which runs `publishRun` when the run was published and has ended, so origin never hands out a run its owner finished. A publish drains the run's records first, so none lands after its end. The drain (`publishPendingRecords`) drops the records of a run whose snapshot on origin names another install, so an install that ends its copy of a run it lost never replaces the owner's record. Once the end is on origin, a repeat writes nothing, so an end that a process saved but stopped before publishing reaches origin on the next request. A claim changes steps only through `takeOver` in `step-state.ts`: it releases a lease, and repends a lease in a `new` step so its next pass gets a new branch. `handoff.ts` never imports the engine. The engine calls `claimRun` for the `claim` field of `next`, and `workflow publish --sessions` passes the engine's `resumableSessions` in. What the snapshot carries is a rule table per type in `snapshot.ts` (`publish`, `drop`, or `rewrite`), so a new run or step field does not compile until it has a rule, and `parseSnapshot` refuses a field it would drop. -- **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. +- **The end reached first wins.** A reported step's `settledAt` is the revision its report made. Every terminal transition, including a skip or cancellation, carries `settledAt`. A step the engine settles takes the run's current revision, which is that of the report that caused it, because every earlier revision was reconciled before that report was accepted. An open lease it fails takes the revision it was leased at, and a cancellation the winning end's revision unless it keeps an already completed agent settlement. `reportAttempt` accepts a report only after `reconcileRun` has applied every end reached before it, so an end a later report reaches never competes with an earlier one. Every end `settleScope` has yet to apply was reached at the run's current revision, and it applies the one declared first. The winner therefore does not depend on when `next` runs between reports. - **A break is a nameless succeeded end.** A succeeded step whose `break` holds competes with the steps that reached an end by the same rule. It ends its workflow, or that workflow's loop, as success. Its value cannot change after its step succeeds, because pull rejects `break` inside a step loop's range, where a previous-pass read could change. - **A workflow loop is decided when a round has nothing left.** Once every step of a looping workflow has settled and no end or break was reached, `decideRound` in `loop.ts` evaluates the workflow's `loop.while`. When it is false, the workflow ends with the round's outcome and outputs. Otherwise every step of the workflow repends for the next round, unless the workflow has run `max_iterations` rounds: then it fails `loop_exhausted` (`WorkflowResult.failure`, and `run.failure`, shown in `done`, at the top). A failed step alone does not stop the loop. The round is kept on the step that runs the workflow (`Scope.runner`), so `repend` resets it with that step, or on the run at the top. Pull rejects a step loop in a looping workflow. - **A step waits for every step it reads.** `readsOf` collects the steps that the step's expressions (`stepExpressions`), templates, and `with` read. For a step that runs an inline workflow, it also collects the outer steps that the inline steps read. `skipUnreachable` uses that set to decide readiness. If a new field reads steps, add it to `stepExpressions` or `templatedTexts`. Otherwise the step can read a value before it exists, or an inline workflow can wait forever. A step outside a loop that needs or reads a step inside it also waits until the loop owner is settled (`waitsFor` in `loop.ts`), and is neither skipped nor ready before. @@ -24,7 +26,7 @@ Each rule below spans more than one file. The rule names the code that owns it, - **An expression reads its absolute paths, wherever they are.** `expressionPaths` returns every `inputs` and `steps` path, including those inside function arguments and predicates, and never a relative path such as `.size`, which reads the current element of a predicate. `readsOf`, `skipUnreachable` and the pull read checks use it, so a new kind of expression must return its paths there. - **Expression types are checked in two places, through `typeProblems`.** `parseExpression` rejects what an expression shows by itself, such as `x < 'a'` or `any(...) < 1`. `readDiagnostics` checks paths against the types the workflow declares. It checks each path only in the workflow that owns it, so a predicate an inline step runs over an enclosing step is reported once. - **Pull and the engine share one list of templated fields, one list of step expressions, and one list of actions.** The templated fields are in `templatedTexts` and the expressions, such as `if` and `break`, in `stepExpressions`, both in `template.ts`. HITL declarations normalize in `parse-hitl.ts`. The actions are in `STEP_ACTIONS` in `types.ts`. Add a new field or action there, not to a copy in `parse.ts` or `engine.ts`. Pull, the cache's `prompt_file` check, the lease render and `readsOf` all split a template with `parseTemplate`, which decides which `{{ … }}` is a template, and the render evaluates it with `evaluateExpression`, as for an `if`. `parseReads` in `engine.ts` and the step parse in `parse.ts` key a table by each expression, so a new one does not compile until both say how to read it. -- **A new step status must fit `stopped()`, `settled()` and `stopWhere()`.** All three are in `step-state.ts`. `stopped()` decides which dependents are skipped, which steps can recover, and the run outcome. `settled()` decides readiness. `stopWhere()` decides what an end or a break does to each step under it. Each switches over every status, so a status added to `StepStateSchema` does not compile until all three place it. +- **A new step status must fit five switches and two predicates.** `stopped()`, `settled()`, `takeOver()` and `stopWhere()` are in `step-state.ts`, and `retryInstance()` in `run-controls.ts`. `stopped()` decides which dependents are skipped, which steps can recover, and the run outcome. `settled()` decides readiness. `takeOver()` decides what a claim does to each step, `stopWhere()` what an end or a break does to each step under it, and `retryInstance()` what a retry does to each step in its cone. The predicates `inProgress()` and `cancellable()` in `step-state.ts` decide whether `settleScope` waits on a step and whether `cancelRun` may stop it. Each of the seven switches over every status with a `never` default, so a status added to `StepStateSchema` does not compile until all of them place it. `StepStateSchema` and the snapshot's `StepCheckpointSchema` are built from one variant list, `SHARED_VARIANTS`, so the status also reaches a handed-over run; they differ only in the `leased` variant. - **Only the transitions in `step-state.ts` write step fields.** `waitForAnswer`, `answerHitl`, `resetPending`, `resetIteration`, `skip`, `lease`, `release`, `start`, `succeed`, `fail`, `cancel`, `repend`, `nextRound`, `endLoop` and `exhaust` change a step in place and drop the fields its new status does not have. `readRun` parses every step through `StepStateSchema`, which gives a run file from an older teamai its defaults: `settledAt` 0, `outcome` from `status`, and no fields its status does not have. - **`run.steps` is replaced, never changed in place.** Its type is `readonly`, so `addItems`, `insertChildren` and `archiveUnder` add or remove instances by assigning a new array. An instance's `id` is `readonly` too, so an instance cannot move to another scope's group. `statesIn` in `scope.ts` keeps the instances of every scope per array, in a `WeakMap`, so settling a scope reads its own instances without scanning every step. A new array starts a new index, except that `insertChildren`, which adds the instances of a workflow a step starts, carries the old index over. - **A pull check that changes what a file means carries a `code`.** `dropsKeptVersion` in `cache.ts` removes the cached copy when a diagnostic has a `code`. An older definition then cannot keep running with the new meaning. A diagnostic without a `code` keeps the last valid version, but only after `checkReferences` checks that version again against what the workflows it runs settle to in the same sync. @@ -43,4 +45,4 @@ Each rule below spans more than one file. The rule names the code that owns it, ## Tests -Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass `workflowConformanceScenarios` in `src/workflow/conformance.ts`, which `runConformanceScenario` in `src/workflow/conformance-interpreter.ts` runs, and a driver that takes workspace leases the git scenarios in `conformance.ts` too. That one list takes in the scenarios a concern keeps in its own file, such as `limits-conformance.ts`, so a runner cannot miss one. If a driver can see a new behavior, add a scenario to the file for its concern; a new file joins the list in `conformance.ts`. +Test parsing through `parseWorkflow` and `syncWorkflowDefinitions`. Test the engine through `startRun`, `leaseNext`, and `reportAttempt`. Every driver must pass `workflowConformanceScenarios` in `src/workflow/conformance.ts`, which `runConformanceScenario` in `src/workflow/conformance-interpreter.ts` runs, and a driver that takes workspace leases the git and delivery scenarios in `conformance.ts` too. That one list takes in the scenarios a concern keeps in its own file, such as `limits-conformance.ts`, so a runner cannot miss one. `workflow-delivery-drivers.test.ts` runs the delivery scenarios through the built CLI for the local driver, a driver that pushes from its own worktree, one that knows nothing of delivery, and one that reports its own pull request. If a driver can see a new behavior, add a scenario to the file for its concern; a new file joins the list in `conformance.ts`. diff --git a/docs/guide/workflow-advanced.md b/docs/guide/workflow-advanced.md index b40d18691..f0949e322 100644 --- a/docs/guide/workflow-advanced.md +++ b/docs/guide/workflow-advanced.md @@ -715,11 +715,11 @@ teamai workflow cancel [step] teamai workflow resume --tools default=claude ``` -`retry` accepts a leased, failed, or cancelled step, a refused approve with an answer, or an answered choose. Retry a HITL request to change its answer; its old answer stays in pass history. It frees an old lease and starts a new pass at a new revision. The local driver stops a command it was running and starts the new pass only after that command has exited. It resets the steps that depend on or read that step, including loop waits and item `after` dependencies. A HITL request among them that waits, or was answered and has not run, is asked again at a new pass. It reopens settled enclosing workflows, items, fan-out steps, and the run. If the reopened run is still running, the result includes a `resume` command. Run it to continue locally, or let your driver poll `next`. Unanswered skips in each affected container are re-derived; manual cancellations outside it stay cancelled. An independent standing end or break prevents retry. +`retry` accepts a leased, failed, or cancelled step, a refused approve with an answer, or an answered choose. Retry a HITL request to change its answer; its old answer stays in pass history. It frees an old lease and starts a new pass at a new revision. The local driver stops a command it was running and starts the new pass only after that command has exited. It resets the steps that depend on or read that step, including loop waits and item `after` dependencies. A HITL request among them that waits, or was answered and has not run, is asked again at a new pass. It reopens settled enclosing workflows, items, fan-out steps, and the run. A step whose pull request delivery failed keeps its pass, and only its delivery runs again. If the reopened run is still running, the result includes a `resume` command. Run it to continue locally, or let your driver poll `next`. Unanswered skips in each affected container are re-derived; manual cancellations outside it stay cancelled. An independent standing end or break prevents retry. History stays in `passes` and the archive, so `length(steps.x.passes)` counts failed attempts too. Retrying a loop owner resets its iteration count. Retrying another step in its range keeps the owner's count. Retry keeps the top-level workflow's round count, so retry in its exhausted final round can fail `loop_exhausted` again. -A single-step `cancel` stops pending, waiting, leased, or running work. Its open descendants stop too. Cancelled steps count as stopped, so dependents skip unless an `if` reads their outcome for recovery. The run normally fails. A whole-run cancel ends immediately with run outcome `cancelled`, without starting another loop round. Cancel does not bump the revision. A late report returns `lease_cancelled`; `next.cancelled` tells drivers which tools to stop. +A single-step `cancel` stops pending, waiting, leased, running, or delivering work. Its open descendants stop too. Cancelled steps count as stopped, so dependents skip unless an `if` reads their outcome for recovery. The run normally fails. A whole-run cancel ends immediately with run outcome `cancelled`, without starting another loop round. Cancel does not bump the revision. A late report returns `lease_cancelled`; `next.cancelled` tells drivers which tools to stop. `resume` drives the run locally after a retry, response, Ctrl-C, or closed terminal. `workflow start --driver local` prints the run id on stderr when the run starts, with the `resume` command that continues it. It uses this machine's open local lease workspace first, then the workspace of the latest driver that called `next` when that driver was local on this machine, then the current directory. Resuming after a retry from a subdirectory therefore keeps the original local workspace. Only one local driver can drive a run at a time. A second resume returns `driver_running` with the owning process id; stop that driver or wait. Locks from dead processes are reclaimed automatically. A settled run returns `done`. Failed and cancelled local runs exit 1; successful control commands exit 0 even when their answer describes a failed or cancelled run. diff --git a/docs/guide/workflow.md b/docs/guide/workflow.md index c94439dca..f72b1cd9b 100644 --- a/docs/guide/workflow.md +++ b/docs/guide/workflow.md @@ -173,7 +173,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. `used` (`{"skills":["review-notes"]}`) names the team skills the pass loaded, for its [scorer labels](scorers.md#any-driver). A value that is not a skill name is dropped, and a `used` of another shape returns `invalid_report`. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. +`outcome` is `success`, `failed`, or `released`. `released` puts the step back to pending. It does not use up a retry. A report for an attempt that was already settled returns `repeated_report`, even when the revision is old. The run keeps recent settled attempts for that check. An older attempt uses the other checks: an old revision returns `stale_revision`, and a current revision that is not the leased attempt returns `wrong_attempt`. Another driver returns `wrong_driver`. `changes` whose `branch` or `base` differs from the lease returns `invalid_report`, and the step stays leased. A read-only step in a workspace reports `read` (`branch`, `base`, and the `commit` it read) in place of `changes`. `changes` from it, a success without `read`, `read` from any other step, or `read` that differs from the lease returns `invalid_report` too. `used` (`{"skills":["review-notes"]}`) names the team skills the pass loaded, for its [scorer labels](scorers.md#any-driver). A value that is not a skill name is dropped, and a `used` of another shape returns `invalid_report`. A payload field the CLI does not read, such as a misspelled `tokens` written as `tokenz` or a field from a newer protocol version, is accepted but reported back: the success envelope of `next`, `report`, and `respond --json` carries `ignored` with those names — top-level fields first, then `driver.` and `changes.` names — names only and never values. The key is left out when nothing was ignored, and an error envelope stays as it is. The run is stored in this scope's TeamAI data directory. The command exits when it finishes. A local run that ends `failed` or `cancelled` exits with status 1. No workflow process keeps running. @@ -216,6 +216,63 @@ steps: command: node -e "require('fs').writeFileSync('joined.txt', 'J')" ``` +## Deliver a PR + +Set `deliver: pr` on a step that writes a branch, and TeamAI opens a pull request from that branch once the step succeeds. It never merges it. A step writes a branch when it has `workspace: new` or `continue`, or when it is a prompt inside a workflow that opened one. A command that delivers must set `workspace`, since a command without one is read-only. A `workflow` step with `workspace: new` delivers its branch once, for all its steps. A step inside it that writes a branch of its own needs a later step that merges that branch through `needs`, or its own `deliver: pr`. Otherwise the delivery fails with `delivery_failed`, since the pull request would leave that work out. On a `for_each` step, each item delivers its own branch. + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +A `deliver: pr` step that would run read-only fails validation, and so does a delivering `workflow` step, `for_each` or not, when no step of its workflow writes a branch. `workflow start` refuses a run that would start from a detached HEAD, which has no branch for a pull request to go into. Check out a named branch first. + +When the step succeeds, its driver has already pushed the branch, as for any writable step, and the step is `delivering`. Steps that need it wait. The next `teamai workflow next`, from whichever driver calls it, looks for an open pull request from that branch into its target, and opens one only when there is none. While another step that writes the same branch still runs, delivery waits for its report, so TeamAI checks that step's commit before the pull request is opened or taken. A run that a crash stopped finds the pull request it opened before and opens no other. So does a run another machine took over, unless the machine that handed it over is still opening one at that moment. GitHub and GitLab reject such a duplicate. TeamAI opens pull requests on GitHub, GitLab, TGit, and GitCode origins, with the login the provider's CLI or token already has, and never starts a login prompt. On any other origin, such as a generic git host or CNB, it opens none, and the step fails naming its branch. + +A pull request goes into the nearest branch its own branch grew from that already delivered one, or else into the branch the run started from. A branch grows first from its `for_each` item's `after` branches, then from its `needs` in order. The first step of a sub-workflow grows from the needs of the step that runs it. A skipped step wrote no branch, so it is passed over. Each pull request then shows only the work done since the branch it goes into. + +Each pass of a looped `workspace: new` step has its own `/p` branch, so it opens its own pull request. The earlier ones stay open, by design. A step that loops on one `continue` branch has one pull request. + +Before it opens or accepts a pull request, TeamAI compares the delivered commit with the run's base commit. A change to CI configuration (such as `.github/workflows` or `.gitlab-ci.yml`), to hooks, to `AGENTS.md` or `CLAUDE.md` at any depth, or to any tool's agent, rule, or MCP files fails the step with `protected_path`, and the message names the files. These are the agent files a workspace step gets from the run's base, plus CI and hook paths. Deletions and renames count, and so do changes the branch got from an earlier step. `teamai workflow retry ` then runs the step again, in a new pass. When only a later commit that another step recorded on a shared branch made the change, the step fails with `delivery_failed` instead, and the retry does not run it again. Its message says how to deliver only the step's own commit, or how to run the steps of that branch again in a new one. + +Any other delivery failure fails the step with `delivery_failed`. Its message says what went wrong, names the branch the work is kept in, and gives the retry command. Fix the cause, such as the provider's credentials, then run `teamai workflow retry `, and run `teamai workflow resume ` or let your driver call `next`. The retry delivers the same branch again, without running the step again. A run cancelled while its pull request is being opened cannot take that pull request back. The run lists it, and the step stays cancelled. + +If origin has no branch to deliver, TeamAI opens no request. The failure names the missing branch and the commit to push to it, then gives the retry command. + +If a host may have opened the request but TeamAI cannot confirm it, the step fails with `delivery_failed` and records no URL. Run the retry command in the message. TeamAI looks for the request before opening one again. + +A driver that opens its own pull request, such as one on another machine, reports it as `changes.pr_url` with the step's other `changes`. TeamAI keeps an `https` URL without credentials, of at most 512 characters, and removes its query and fragment. On the origin's own host, the URL must be the open pull request of the branch into its target, and one TeamAI cannot check with the host is dropped. Delivery then finds the open pull request itself. A URL on another host, such as a private review service, is kept as it is. A URL TeamAI does not keep is dropped, and so is one no delivery uses: from a step without `deliver: pr`, from a step that failed or was released, or for a branch with no changes beyond the run base. The report is still accepted, and its envelope lists `"dropped":[{"field":"changes.pr_url","reason":"…"}]` without the value. The branch is then delivered as for any driver. + +### Start from CI + +A GitHub Actions job can start a run with the local driver, and the run delivers its pull requests unattended. Store the workflow in the team repo. Give the job a token that can read the team repo, push branches to the product repo, and open pull requests there. The job below reads it from a `WORKFLOW_TOKEN` secret, and the team repo's URL from a `TEAM_REPO` variable. The runner also needs the tool that runs the prompt steps, signed in, and `--tools` picks it for this start. + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --tools default=claude --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` answers an optional `approve` request yes and an optional `choose` request with its default. Nobody can answer a required one in an unattended job, so its step fails with `hitl_required`, and so does the run. To start a run from another branch, such as one whose CI failed, check that branch out by name, for example `ref: ${{ github.head_ref }}` on a `pull_request` event. The default `pull_request` checkout is a detached merge commit, which `workflow start` refuses. TeamAI installs no trigger and does not wait for CI or a merge. + ## Continue a run on another machine A run lives in the TeamAI data directory of the machine that started it. To continue it somewhere else, publish it, then resume it on the other machine. Use this when your tool runs out of tokens in the middle of a run, when you hand a run to a teammate, or when a CI run failed and you want to finish it on your own machine. @@ -239,10 +296,16 @@ A step with `session: continue` or `session.from` starts a new agent session on With [usage reporting](reporting.md) on, every run publishes a privacy-preserving record to the team's `teamai-reports` branch: once when it starts and once when it ends. With it off, nothing is published and the envelopes carry no `record`. Records live in `workflows/runs/.json`, keyed by run id, so a retried publication updates the same record instead of adding a copy. -A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. +A record carries only: the run id, workflow name, `sourceHash`, optional `teamRevision`, phase (`start` or `end`), driver kind, status, the end the run reached (when it reached one), the failure code of a run that failed without reaching an end, when it is a code teamai sets, such as `loop_exhausted` for a loop that ran out or `tool_unavailable` when the local driver cannot find the tool (never its message), start and end times, duration, per-step entries (step path such as `check.classify`, agent, tool, model, outcome, tokens — a mixed-agent run records each step's own fields; `tool` is kept only when it is one of the local driver's tools, and `model` is the name the driver reported, kept only when it is shaped like a model name), summary step counts (succeeded, failed, skipped, cancelled), the cumulative settled-attempt count, the pull requests the run delivered (`prs`), and the publishing username. `sourceHash` identifies the exact workflow and prompt bytes that ran. `teamRevision` is present only when one remote-reachable team commit stores those exact bytes for the parent and every child workflow; local or unpushed edits and mixed cached revisions leave it out. Every entry has its `pass` number, and a step that a loop ran more than once has one entry per pass. An item of a `for_each` counts the times that item ran, so its `pass` can be lower than its step's when an earlier pass had fewer items. A cancelled step's entry has the outcome `cancelled`, and a skipped step's has none. The counts cover each step once, as the run ended. Steps of a sub-workflow that never started are not counted. Prompts, command text, input values, `env` values, outputs, diffs, secret values, local workspace paths, session ids, and hostnames have no field to land in — the record is built from an allowlist, not filtered from the run state. An entry's `agent` is what that pass ran as, as observed. It holds the step's `agent:` definition and how the tool got it (`main` when the tool reads its own agent file, `prompt` when the instructions are prepended), the tool and its version, the model and effort the launch asked for and the model the tool reported, the team commit the checkout last pulled and whether a delivered file was edited since, the project commit, the TeamAI version, and the driver kind. A value nobody observed is left out. `fingerprint` hashes the definition, tool, model, effort and team commit, so two passes with the same fingerprint ran with the same recorded parts. It is left out when the model or the team commit is unknown, or a delivered file was edited. A pass nothing observed, and a record from an older TeamAI, carry only the definition's name. While a step runs, the tool it starts, and any session that tool starts in the same checkout, skips the session-start pull when the checkout's last pull delivered completely to that tool, so the step keeps the team commit it records. +`prs` lists every pull request the run delivered, whether TeamAI opened it or a driver reported it, once each, in the order the run first delivered them. A URL from a step's output is never listed. Like the rest of the record, `prs` follows the usage-report opt-out, and it is public when the team repo is public. A record from before `prs` still reads. To find the run that opened a pull request, run this in a checkout of the team repo's `teamai-reports` branch: + +```text +jq -r 'select((.prs // []) | index("")) | .run' workflows/runs/*.json +``` + The operation that settles the run queues its end record under the run lock and publishes it after releasing that lock. Retry updates the same record path when the run ends again. Answer entries publish choice, channel, and agent only; notes, answer timestamps, and answer driver identities stay local. A publication that fails — no remote, a busy reports lock, a rejected push — is queued in this scope's data directory and retried on the next `teamai pull`. The `start` and `next` envelopes then carry `record: pending` instead of `record: published`. The run itself is unaffected: a machine that dies after starting a run leaves it visible as started but not ended. Run ids are `r` plus 16 hex digits. Runs created before this scheme used 8 hex digits and still resolve. diff --git a/docs/guide/zh-CN/workflow-advanced.md b/docs/guide/zh-CN/workflow-advanced.md index 73dd64fb4..308d2a20a 100644 --- a/docs/guide/zh-CN/workflow-advanced.md +++ b/docs/guide/zh-CN/workflow-advanced.md @@ -715,11 +715,11 @@ teamai workflow cancel [step] teamai workflow resume --tools default=claude ``` -`retry` 接受 leased、failed 或 cancelled 步骤，也接受已回答并拒绝的 approve 和已回答的 choose。重试 HITL 请求可以更改答案，旧答案保留在 pass 历史中。它释放旧租约，以新修订号启动新 pass（本地驱动会停止正在运行的命令，并等它退出后才启动新 pass），重置依赖或读取该步骤的后续步骤，包括循环等待和 item 的 after 依赖（其中正在等待、或已回答但尚未运行的 HITL 请求会在新 pass 中重新询问），并重新打开已结算的外层工作流、item、fan-out 步骤和运行。如果重新打开后运行仍在进行，返回结果会包含 `resume` 命令。执行该命令继续本地运行，或让驱动轮询 `next`。每个受影响容器中未回答的跳过步骤会重新计算；范围外手动取消的步骤保持取消。无关步骤已到达的有效结局或 break 会阻止重试。 +`retry` 接受 leased、failed 或 cancelled 步骤，也接受已回答并拒绝的 approve 和已回答的 choose。重试 HITL 请求可以更改答案，旧答案保留在 pass 历史中。它释放旧租约，以新修订号启动新 pass（本地驱动会停止正在运行的命令，并等它退出后才启动新 pass），重置依赖或读取该步骤的后续步骤，包括循环等待和 item 的 after 依赖（其中正在等待、或已回答但尚未运行的 HITL 请求会在新 pass 中重新询问），并重新打开已结算的外层工作流、item、fan-out 步骤和运行。交付 pull request 失败的步骤保留原来的 pass，只重新运行交付。如果重新打开后运行仍在进行，返回结果会包含 `resume` 命令。执行该命令继续本地运行，或让驱动轮询 `next`。每个受影响容器中未回答的跳过步骤会重新计算；范围外手动取消的步骤保持取消。无关步骤已到达的有效结局或 break 会阻止重试。 历史保留在 passes 和 archive 中，因此 `length(steps.x.passes)` 也统计失败尝试。重试循环所有者会重置其迭代数；重试范围内的其他步骤则保留所有者的计数。重试保留顶层工作流的 round，耗尽最后一轮后重试仍可能再次 `loop_exhausted`。 -指定步骤的 cancel 停止 pending、waiting、leased 或 running 工作，同时停止尚未结束的后代。cancelled 属于 stopped，因此依赖步骤会跳过，除非其 if 读取取消结果来恢复；运行通常失败。取消整个运行立即以 `cancelled` 结束，不启动下一轮。cancel 不增加修订号。迟到的报告返回 `lease_cancelled`，`next.cancelled` 告诉驱动停止哪些工具。 +指定步骤的 cancel 停止 pending、waiting、leased、running 或 delivering 工作，同时停止尚未结束的后代。cancelled 属于 stopped，因此依赖步骤会跳过，除非其 if 读取取消结果来恢复；运行通常失败。取消整个运行立即以 `cancelled` 结束，不启动下一轮。cancel 不增加修订号。迟到的报告返回 `lease_cancelled`，`next.cancelled` 告诉驱动停止哪些工具。 在重试、回答、Ctrl-C 或关闭终端后，用 resume 继续本地运行。`workflow start --driver local` 在运行开始时就把运行 id 和继续它的 `resume` 命令输出到 stderr。它优先使用本机未结本地租约的工作目录，其次使用最近调用 `next` 的驱动工作目录（该驱动必须是本机本地驱动），最后使用当前目录。因此重试后即使从子目录继续，也会保留原来的本地工作目录。每个运行同时只能有一个本地驱动。重复 resume 返回 driver_running 和持有者进程号，请停止该驱动或等待；死进程留下的锁会自动回收。已结算运行返回 done。本地运行以 failed 或 cancelled 结束时退出码为 1；控制命令成功时退出码为 0，即使返回的运行状态为 failed 或 cancelled。 diff --git a/docs/guide/zh-CN/workflow.md b/docs/guide/zh-CN/workflow.md index 43062c079..744252683 100644 --- a/docs/guide/zh-CN/workflow.md +++ b/docs/guide/zh-CN/workflow.md @@ -173,7 +173,7 @@ teamai workflow next --json '{"driver":{"kind":"local","machine":"host","w teamai workflow report --json '{"attempt":"","revision":1,"outcome":"success","driver":{"kind":"local","machine":"host","workspace":"/work"}}' ``` -`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。`used`（`{"skills":["review-notes"]}`）写明该轮次加载过的团队技能，供它的[评分标签](scorers.md#任意-driver)使用。不是技能名的值会被丢弃，其他形状的 `used` 返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 +`outcome` 可以是 `success`、`failed` 或 `released`。`released` 把步骤退回 pending，不消耗重试。已经上报过的 attempt 返回 `repeated_report`，revision 旧了也一样。运行状态会记住最近上报过的 attempt。更早的 attempt 走其余检查：revision 旧了返回 `stale_revision`，revision 是当前值但不是当前租约的 attempt 时返回 `wrong_attempt`。其他驱动返回 `wrong_driver`。`changes` 的 `branch` 或 `base` 与租约不符时返回 `invalid_report`，步骤仍处于租约中。工作区里的只读步骤上报 `read`（`branch`、`base` 和它读取的 `commit`），而不是 `changes`。它上报 `changes`、成功时没有 `read`、其他步骤上报 `read`，或 `read` 与租约不符时，同样返回 `invalid_report`。`used`（`{"skills":["review-notes"]}`）写明该轮次加载过的团队技能，供它的[评分标签](scorers.md#任意-driver)使用。不是技能名的值会被丢弃，其他形状的 `used` 返回 `invalid_report`。CLI 不读取的 payload 字段——例如把 `tokens` 拼成 `tokenz`，或来自更新协议版本的字段——会被接受但回报出来：`next`、`report` 和 `respond --json` 的成功信封会带上 `ignored`，先列顶层字段、再列 `driver.<字段>` 和 `changes.<字段>`，只有名字、没有值。没有未读字段时不带这个键，错误信封保持原样。 运行状态保存在当前 scope 的 TeamAI 数据目录里。命令结束后进程就退出。本地运行以 `failed` 或 `cancelled` 结束时，退出码是 1。不会留下常驻的工作流进程。 @@ -216,6 +216,63 @@ steps: command: node -e "require('fs').writeFileSync('joined.txt', 'J')" ``` +## 交付 PR + +在写分支的步骤上设置 `deliver: pr`，步骤成功后 TeamAI 会从这条分支开一个 pull request，并且从不合并它。设置了 `workspace: new` 或 `continue` 的步骤会写分支，位于已开分支的工作流中的提示词步骤也会。交付的命令必须设置 `workspace`，因为没有 `workspace` 的命令是只读的。设置了 `workspace: new` 的 `workflow` 步骤为它的全部步骤只交付一次自己的分支。其中写自己分支的步骤，需要有后续步骤通过 `needs` 合并那条分支，或者自己设置 `deliver: pr`。否则交付以 `delivery_failed` 失败，因为 pull request 会漏掉那部分工作。`for_each` 步骤的每个 item 各自交付自己的分支。 + +```yaml +schemaVersion: v1alpha1 +name: ship-issue +description: Implement an issue for review. +inputs: { issue: number } +steps: + - id: change + workspace: new + deliver: pr + prompt: "Implement issue #{{ inputs.issue }}." +``` + +会以只读方式运行的 `deliver: pr` 步骤无法通过校验；设置了 `deliver: pr` 的 `workflow` 步骤（无论是否 `for_each`），若其工作流中没有步骤写分支，也无法通过校验。运行若会从 detached HEAD 开始，`workflow start` 会拒绝它，因为 pull request 没有可以合入的分支。请先检出一个有名称的分支。 + +步骤成功时，驱动已经像对任何可写步骤一样推送了分支，步骤进入 `delivering` 状态，需要它的步骤会等待。下一次 `teamai workflow next`（无论哪个驱动调用）会查找从这条分支合入目标分支的已打开 pull request，没有时才新开一个。写同一条分支的另一个步骤仍在运行时，交付会等它上报，这样 TeamAI 会在开出或接受 pull request 之前检查那个步骤的提交。因崩溃而中断的运行，会找到之前开的 pull request，不会再开第二个。被另一台机器接管的运行也是如此，除非交出运行的机器此刻仍在开 pull request；GitHub 和 GitLab 会拒绝这样的重复请求。TeamAI 在 GitHub、GitLab、TGit 和 GitCode 的 origin 上开 pull request，使用 provider 的 CLI 或 token 已有的登录，从不弹出登录提示。在其他 origin 上（例如通用 Git 主机或 CNB），它不开 pull request，步骤失败并写明分支。 + +pull request 合入这条分支来源中最近一条已经交付过的分支，没有时合入运行开始时的分支。分支首先来自其 `for_each` item 的 `after` 分支，然后依次来自 `needs`。子工作流的第一个步骤来自运行它的步骤的 needs。跳过的步骤没有写分支，会被越过。因此每个 pull request 只显示它所合入的分支之后的工作。 + +循环中 `workspace: new` 步骤的每个 pass 都有自己的 `/p` 分支，因此各开一个 pull request。之前的 pull request 保持打开，这是有意为之。在同一条 `continue` 分支上循环的步骤只有一个 pull request。 + +开出或接受 pull request 之前，TeamAI 将交付的提交与运行的基准提交比较。修改 CI 配置（例如 `.github/workflows` 或 `.gitlab-ci.yml`）、hooks、任意层级的 `AGENTS.md` 或 `CLAUDE.md`，或任一工具的 agent、rule、MCP 文件，步骤会以 `protected_path` 失败，消息列出这些文件。这些路径就是工作区步骤从运行基准获得的 agent 文件，再加上 CI 和 hook 路径。删除和重命名也算，分支从前序步骤带来的修改也算。之后运行 `teamai workflow retry `，会在新的 pass 中重新运行这个步骤。如果修改只来自另一个步骤之后在共享分支上记录的提交，步骤改为以 `delivery_failed` 失败，重试也不会重新运行它。消息会说明如何只交付该步骤自己的提交，或如何在新分支上重新运行那条分支的步骤。 + +其他交付失败会让步骤以 `delivery_failed` 失败。消息说明出了什么问题、工作保存在哪条分支，并给出重试命令。修复原因（例如 provider 的凭据）后，运行 `teamai workflow retry `，再运行 `teamai workflow resume `，或让驱动调用 `next`。重试会再次交付同一条分支，不会重新运行步骤。运行在开 pull request 期间被取消时，这个 pull request 无法收回。运行会列出它，步骤保持 cancelled。 + +如果 origin 没有要交付的分支，TeamAI 不会打开请求。失败消息会指出缺失的分支和要推送到该分支的提交，然后给出重试命令。 + +如果主机可能已打开请求，但 TeamAI 无法确认，步骤会以 `delivery_failed` 失败，且不记录 URL。运行消息中的重试命令。TeamAI 会先查找已有请求，再尝试打开请求。 + +自己开 pull request 的驱动（例如在另一台机器上的驱动）把它作为 `changes.pr_url`，和步骤的其他 `changes` 一起上报。TeamAI 保留不含凭据、不超过 512 个字符的 `https` URL，并去掉 query 和 fragment。在 origin 自己的主机上，URL 必须是这条分支合入目标分支的已打开 pull request；TeamAI 无法向主机核对的 URL 会被丢弃，之后由交付自己查找已打开的 pull request。其他主机上的 URL（例如私有审查服务）原样保留。TeamAI 不保留的 URL 会被丢弃；没有交付使用的 URL 也会被丢弃：来自未设置 `deliver: pr` 的步骤、来自失败或被 released 的步骤，或分支相对运行基线没有变更时的 URL。report 仍会被接受，其信封列出 `"dropped":[{"field":"changes.pr_url","reason":"…"}]`，不包含 URL 本身。随后这条分支像其他驱动的分支一样交付。 + +### 从 CI 启动 + +GitHub Actions 作业可以用本地驱动启动运行，运行会在无人值守的情况下交付它的 pull request。把工作流放进团队仓库。给作业一个 token，它要能读取团队仓库、向产品仓库推送分支，并在产品仓库开 pull request。下面的作业从 `WORKFLOW_TOKEN` secret 读取它，从 `TEAM_REPO` 变量读取团队仓库的 URL。runner 还需要运行提示词步骤的工具，并且已经登录，`--tools` 为这次启动选择它。 + +```yaml +on: { issues: { types: [labeled] } } +concurrency: ship-issue-${{ github.event.issue.number }} +jobs: + ship: + runs-on: ubuntu-latest + env: + TEAM_REPO: ${{ vars.TEAM_REPO }} + GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }} + steps: + - uses: actions/checkout@v5 + with: { ref: main, token: "${{ secrets.WORKFLOW_TOKEN }}" } + - run: npx teamai-cli init "$TEAM_REPO" --scope workspace + - run: npx teamai-cli pull + - run: npx teamai-cli workflow start ship-issue --driver local --tools default=claude --auto-approve --inputs '{"issue":${{ github.event.issue.number }}}' +``` + +`--auto-approve` 对可选的 `approve` 请求回答 yes，对可选的 `choose` 请求选择默认值。无人值守的作业里没人能回答必需的请求，因此它所在的步骤以 `hitl_required` 失败，运行也随之失败。要从另一条分支启动运行（例如 CI 失败的分支），请按分支名检出它，例如在 `pull_request` 事件中使用 `ref: ${{ github.head_ref }}`。默认的 `pull_request` checkout 是 detached merge commit，`workflow start` 会拒绝它。TeamAI 不安装触发器，也不等待 CI 或合并。 + ## 在另一台机器上继续运行 运行保存在启动它的那台机器的 TeamAI 数据目录里。要在别处接着运行，先发布它，再在另一台机器上继续。适用的情况：工具在运行中途用完了 token，把运行交给队友，或者 CI 里的运行失败了，你想在自己的机器上把它跑完。 @@ -239,10 +296,16 @@ teamai workflow resume # 在另一台机器上，位于同一个仓 开启[使用统计上报](reporting.md)时，每次运行都会向团队的 `teamai-reports` 分支发布一条保护隐私的记录：开始时一次，结束时一次。关闭后不发布任何记录，返回中也不带 `record`。记录存放在 `workflows/runs/.json`，以 run id 为键，重试的发布会更新同一条记录而不是产生副本。 -记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数，以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 +记录只包含：run id、工作流名、`sourceHash`、可选的 `teamRevision`、阶段（`start` 或 `end`）、驱动的 kind、状态、运行到达的 end（到达时才有）、未到达 end 就失败的运行的失败代码（仅限 teamai 设定的代码，如循环用完次数时的 `loop_exhausted`，或本地驱动找不到工具时的 `tool_unavailable`，不含失败消息）、开始和结束时间、时长、逐步条目（step 路径如 `check.classify`、agent、tool、model、outcome、tokens——混合 agent 的运行会分别记录每步自己的字段；`tool` 仅在是本地驱动的工具之一时保留，`model` 是驱动报告的名称，仅在形如模型名时保留）、步骤汇总计数（succeeded、failed、skipped、cancelled）、累计已结算 attempt 数、运行交付的 pull request（`prs`），以及发布者用户名。`sourceHash` 标识实际运行的工作流和提示文件的精确字节。只有一个远端可达的团队提交同时保存父工作流和所有子工作流的这些精确字节时，才写入 `teamRevision`；本地或未推送的修改以及混合缓存版本都会省略它。每条条目都带 `pass` 编号，循环运行了多次的步骤每个 pass 各有一条条目。`for_each` 的项按该项自己运行的次数计 `pass`，所以前面某个 pass 的项较少时，项的 `pass` 可能比所属步骤的小。被取消步骤的条目 outcome 为 `cancelled`，被跳过的步骤没有 outcome。汇总计数按运行结束时的状态，每个步骤只计一次。从未开始的子工作流里的步骤不计入。提示词、命令文本、输入值、`env` 值、输出、diff、秘密值、本地工作区路径、会话 id 和主机名没有可落地的字段——记录由白名单构建，而不是从运行状态里过滤。 条目的 `agent` 是该 pass 实际运行时观察到的组合：步骤的 `agent:` 定义及工具如何拿到它（工具自己读取 agent 文件时为 `main`，指令拼接到提示词前时为 `prompt`）、工具及其版本、启动时请求的模型和 effort、工具报告的模型、检出最近一次 pull 的团队提交以及之后是否改过已下发的文件、项目提交、TeamAI 版本和驱动的 kind。没有观察到的值不写。`fingerprint` 对定义、工具、模型、effort 和团队提交做哈希，fingerprint 相同的两个 pass 记录下来的这些部分相同。模型或团队提交未知，或改过已下发的文件时，不写 `fingerprint`。没有任何观察的 pass，以及旧版 TeamAI 写的记录，只带定义的名称。步骤运行期间，如果该检出最近一次 pull 已完整下发到步骤启动的工具，该工具以及它在同一检出中启动的会话都会跳过会话开始时的 pull，步骤因此保持它记录的团队提交。 +`prs` 列出运行交付的每个 pull request，无论是 TeamAI 开的还是驱动上报的，每个只列一次，按运行首次交付它们的顺序排列。步骤输出中的 URL 从不列入。和记录的其他部分一样，`prs` 遵循 usage-report 的退出设置，团队仓库公开时它也公开。没有 `prs` 的旧记录仍可读取。要查找开出某个 pull request 的运行，在团队仓库 `teamai-reports` 分支的检出中运行： + +```text +jq -r 'select((.prs // []) | index("")) | .run' workflows/runs/*.json +``` + 结束运行的操作在锁内排队保存结束记录，在释放锁后发布。发布失败时——没有远端、reports 锁被占用、push 被拒——记录会进入当前 scope 数据目录的本地队列，在下一次 `teamai pull` 时重试。此时 `start`、`next` 的返回会带 `record: pending` 而不是 `record: published`。运行本身不受影响：一台机器在运行开始后崩溃，该运行会以"已开始未结束"的状态保持可见。 Run id 是 `r` 加 16 位十六进制。此方案之前创建的运行用 8 位十六进制，仍然可以解析。 diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 92a5623df..272aa7e8f 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -221,7 +221,7 @@ Generated: do not edit by hand. Regenerate with ## workflow -- `teamai workflow` — Run a validated workflow from workflows/.yaml +- `teamai workflow` — Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them. - `teamai workflow start ` — Create a run. With --driver local, execute command and prompt steps in this process. - `--driver ` — Run the workflow in this process. Only local is available. - `--tools ` — Override local prompt tools for this run, as comma-separated name=tool pairs @@ -235,14 +235,14 @@ Generated: do not edit by hand. Regenerate with - `--json ` — Driver identity {"driver":{"kind":"ide","machine":"host","workspace":"/work"}} declares an answer a person gave in the driver UI, never one its agent chose. Records channel driver with agent false; accepts required HITL requests. - `--note ` — Keep a note with the answer, readable as steps..note - `--pass ` — Answer only this positive pass number; rejects an answered or replaced HITL request - - `teamai workflow retry ` — Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope. + - `teamai workflow retry ` — Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. A failed pull request delivery is retried in the same pass, without running the step again. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope. - `teamai workflow resume ` — Continue a run with the local driver after retry, respond, or a stopped driver (Ctrl-C or closed terminal). Uses this machine's open local lease workspace, then the latest local driver's workspace on this machine, then cwd. Another live local driver returns driver_running. A settled run returns done. A run this machine does not have is taken from teamai-reports, where workflow publish put it; a published run continues only while this install owns it, else not_owner names the owner. - `--tools ` — Override local prompt tools for this run, as comma-separated name=tool pairs - `--auto-approve` — Automatically answer optional approve HITL requests and choose defaults in this process - `--claim` — Take the published run over from the install that owns it, replacing this machine's copy and releasing its leases - `teamai workflow publish ` — Publish a run's state to teamai-reports so another member or a runner can continue it with workflow resume. The first publish makes this install the run's owner; publishing an unchanged run writes nothing. Refuses a run with records not yet on teamai-reports, or with a value that looks like a secret. Prints one JSON envelope listing the leases still in flight. - `--sessions` — Also put the agent sessions that the run's unsettled steps continue (session: continue or session.from) in the team session archive, scrubbed, so whoever continues the run resumes them; listed in the envelope as sessions. Without it, those steps start new sessions on another machine - - `teamai workflow cancel [step]` — Cancel a whole run, or a pending, waiting, leased, or running step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope. + - `teamai workflow cancel [step]` — Cancel a whole run, or a pending, waiting, leased, running, or delivering step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope. ## score diff --git a/src/__tests__/e2e/workflow-delivery-drivers.test.ts b/src/__tests__/e2e/workflow-delivery-drivers.test.ts new file mode 100644 index 000000000..b810f6254 --- /dev/null +++ b/src/__tests__/e2e/workflow-delivery-drivers.test.ts @@ -0,0 +1,446 @@ +/** + * Real CLI: the delivery scenarios of `conformance.ts` for each kind of driver, through `teamai workflow` JSON alone. Two + * installs, each with its own HOME and checkout, share a team origin and a product origin whose URL is + * `https://github.com/fixture/app.git`, which git rewrites to a bare repository. One fake `gh` on PATH is the host of + * every case: it records each create, answers lookups from what it recorded, and holds the call a case names. + * + * - local: `workflow start --driver local` runs the run; after a crash, `workflow resume` continues it. While another + * process delivers, `workflow resume` drives a run another driver reported. + * - session: commits in a worktree of the install's checkout, pushes the branch itself, reports, then calls next. + * - external: knows nothing of delivery. It works in a clone of its own and sleeps on every `wait`, whatever it says. + * - remote: works in a clone on another machine, opens its own pull request there, and reports it as changes.pr_url. + */ +import { execFileSync, execSync, spawn, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { deliveryConformanceWorkflow, workflowDeliveryConformanceScenarios, type DeliveryConformanceScenario } from '../../workflow/conformance.js'; +import { fakeGhHost, type FakeGh } from '../helpers/fake-gh.js'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const CLI = path.join(ROOT, 'dist', 'index.js'); +const TEAM_URL = 'https://github.com/acme/team.git'; +const PRODUCT_URL = 'https://github.com/fixture/app.git'; +const POLL_MS = 250; +const HOLD_MS = 2_000; + +type Profile = 'local' | 'session' | 'external' | 'remote'; +type Member = 'alice' | 'bob'; + +const EnvelopeSchema = z.object({ + data: z.record(z.unknown()).optional(), + error: z.object({ code: z.string(), message: z.string() }).passthrough().optional(), +}); +type Envelope = z.infer; + +const LeaseSchema = z.object({ + step: z.string(), attempt: z.string(), revision: z.number(), command: z.string().optional(), + workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }), start: z.object({ commit: z.string() }) }).optional(), +}); +type Lease = z.infer; + +const ReceiptSchema = z.object({ url: z.string(), branch: z.string(), base: z.string(), step: z.string(), pass: z.number() }); +const RunFileSchema = z.object({ + status: z.string(), + steps: z.array(z.object({ + id: z.string(), status: z.string(), + changes: z.object({ branch: z.string(), to: z.string() }).optional(), + failure: z.object({ code: z.string(), message: z.string() }).optional(), + })), + deliveries: z.array(ReceiptSchema).optional(), +}); +type RunFile = z.infer; +const RecordSchema = z.object({ run: z.string(), prs: z.array(z.string()).optional() }); +const OpenSchema = z.array(z.object({ html_url: z.string(), head: z.object({ ref: z.string() }) })); + +interface Result { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +let sandbox = ''; +let teamOrigin = ''; +let productOrigin = ''; +let gh: FakeGh; +let ghDir = ''; +let clones = 0; + +const homeOf = (member: Member): string => path.join(sandbox, member); +const workOf = (member: Member): string => path.join(homeOf(member), 'work'); +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +function envOf(member: Member): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: homeOf(member), USERPROFILE: homeOf(member), GIT_CONFIG_GLOBAL: path.join(homeOf(member), '.gitconfig'), + GH_CONFIG_DIR: path.join(homeOf(member), '.gh'), PATH: `${ghDir}${path.delimiter}${process.env.PATH ?? ''}`, + GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', FORCE_COLOR: '0', + SHIP_OPEN: path.join(ghDir, 'open.json'), SHIP_SAW: path.join(ghDir, 'ship-saw.json'), + }; + for (const name of ['XDG_CONFIG_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'GITHUB_TOKEN', 'GH_TOKEN']) delete env[name]; + return env; +} + +function git(cwd: string, args: string[], env: NodeJS.ProcessEnv = process.env): string { + return execFileSync('git', args, { cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +} + +function teamai(member: Member, args: string[], options: { cwd?: string; spawned?: (child: ChildProcess) => void; stderr?: (text: string) => void } = {}): Promise { + return new Promise((resolve) => { + const child = spawn(process.execPath, [CLI, ...args], { cwd: options.cwd ?? workOf(member), env: envOf(member), stdio: ['ignore', 'pipe', 'pipe'] }); + options.spawned?.(child); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString(); options.stderr?.(stderr); }); + child.on('close', (code, signal) => resolve({ code, signal, stdout, stderr })); + }); +} + +function envelope(result: Result): Envelope { + const line = result.stdout.trim().split('\n').filter((item) => item.startsWith('{')).at(-1); + if (!line) throw new Error(`no JSON envelope:\n${said(result)}`); + return EnvelopeSchema.parse(JSON.parse(line)); +} + +function said(result: Result): string { + return `exit ${result.code ?? result.signal}\n${result.stdout}\n${result.stderr}`; +} + +async function waitFor(check: () => boolean, ms: number): Promise { + for (const until = Date.now() + ms; !check(); await sleep(50)) if (Date.now() > until) return false; + return true; +} + +function seed(dir: string, files: Record, origin: string): void { + fs.mkdirSync(dir, { recursive: true }); + git(dir, ['init', '-q', '-b', 'main']); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), text); + } + git(dir, ['add', '-A']); + git(dir, ['-c', 'user.name=Seed', '-c', 'user.email=seed@example.com', 'commit', '-qm', 'seed']); + git(dir, ['push', '-q', origin, 'main']); +} + +async function join(member: Member): Promise { + fs.mkdirSync(path.join(homeOf(member), '.gh'), { recursive: true }); + fs.writeFileSync(path.join(homeOf(member), '.gitconfig'), [ + '[user]', `\tname = ${member}`, `\temail = ${member}@example.com`, + `[url "file://${teamOrigin}"]`, `\tinsteadOf = ${TEAM_URL}`, + `[url "file://${productOrigin}"]`, `\tinsteadOf = ${PRODUCT_URL}`, '', + ].join('\n')); + git(sandbox, ['clone', '-q', PRODUCT_URL, workOf(member)], envOf(member)); + for (const args of [['init', TEAM_URL, '--provider', 'git', '--agent', 'claude', '--scope', 'user', '--force'], ['pull']]) { + const result = await teamai(member, args); + expect(result.code, said(result)).toBe(0); + } +} + +function readRun(member: Member, run: string): RunFile { + return RunFileSchema.parse(JSON.parse(fs.readFileSync(path.join(homeOf(member), '.teamai', 'workflows', 'runs', `${run}.json`), 'utf8'))); +} + +function endRecordPrs(run: string): string[] | undefined { + const text = git(sandbox, ['--git-dir', teamOrigin, 'show', `teamai-reports:workflows/runs/${run}.json`]); + return RecordSchema.parse(JSON.parse(text)).prs; +} + +function createsOf(run: string): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr' && call.args[1] === 'create' && call.args.includes(`teamai/${run}/work/p1`)) + .map((call) => ['-R', '-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +const ReportAnswerSchema = z.object({ step: z.object({ status: z.string() }) }); + +/** + * What a stub driver did: the steps it was leased, the steps it reported with the status each report answered, its + * next calls, those killed, and the waits it slept on. + */ +interface DriverLog { + leases: string[]; + reports: Array<{ step: string; status: string }>; + nexts: number; + killed: number; + waits: number; +} + +/** + * A driver that speaks only `teamai workflow next|report` JSON. It runs each leased command, in the workspace its + * profile keeps for a lease with one, commits and pushes that branch itself, and reports. It sleeps on every `wait`, and + * calls `next` again when one was killed, unless `stopped` says it was stopped. + */ +async function stubDriver(profile: Exclude, member: Member, run: string, options: { + claim?: boolean; + afterReport?: (step: string) => Promise; + stopped?: () => boolean; + reportedUrl?: (branch: string) => { open: string; report: string }; + onReport?: (step: string, result: Result) => void; +} = {}): Promise { + const log: DriverLog = { leases: [], reports: [], nexts: 0, killed: 0, waits: 0 }; + const driver = { kind: profile, machine: `${member}-${profile}`, workspace: workOf(member), can: { workspace: ['new', 'continue', 'read-only'] } }; + let claim = options.claim === true; + for (let calls = 0; calls < 80; calls++) { + if (options.stopped?.()) return log; + const next = await teamai(member, ['workflow', 'next', run, '--json', JSON.stringify({ driver, ...(claim ? { claim: true } : {}) })]); + log.nexts++; + if (next.signal) { + log.killed++; + continue; + } + claim = false; + const answer = envelope(next); + if (answer.error) throw new Error(`next answered an error:\n${said(next)}`); + if (answer.data?.done) return log; + if (answer.data?.wait) { + log.waits++; + await sleep(POLL_MS); + continue; + } + const lease = LeaseSchema.parse(answer.data?.lease); + log.leases.push(lease.step); + const reported = await work(profile, member, lease, options.reportedUrl); + const body = { attempt: lease.attempt, revision: lease.revision, outcome: 'success', driver, ...reported }; + const result = await teamai(member, ['workflow', 'report', run, '--json', JSON.stringify(body)]); + const answered = envelope(result); + log.reports.push({ step: lease.step, status: ReportAnswerSchema.safeParse(answered.data).data?.step.status ?? answered.error?.code ?? '' }); + options.onReport?.(lease.step, result); + expect(result.code === 0 || answered.error?.code === 'lease_cancelled', said(result)).toBe(true); + await options.afterReport?.(lease.step); + } + throw new Error(`${profile} driver did not finish run ${run}: ${JSON.stringify(log)}`); +} + +/** Runs a lease's command where `profile` keeps its workspace, and returns the `changes` the report carries. */ +async function work(profile: Exclude, member: Member, lease: Lease, reportedUrl?: (branch: string) => { open: string; report: string }): Promise> { + const env = envOf(member); + if (!lease.workspace) { + execSync(lease.command ?? 'true', { cwd: workOf(member), env, stdio: 'ignore' }); + return {}; + } + const { branch, base, start } = lease.workspace; + const dir = path.join(sandbox, `${member}-${profile}-${clones++}`); + if (profile === 'session') git(workOf(member), ['worktree', 'add', '-q', '-b', branch, dir, start.commit], env); + else { + git(sandbox, ['clone', '-q', PRODUCT_URL, dir], env); + git(dir, ['checkout', '-q', '-b', branch, start.commit], env); + } + execSync(lease.command ?? 'true', { cwd: dir, env, stdio: 'ignore' }); + git(dir, ['add', '-A'], env); + git(dir, ['commit', '-qm', lease.step], env); + git(dir, ['push', '-q', 'origin', branch], env); + const to = git(dir, ['rev-parse', 'HEAD'], env); + const changes: Record = { branch, base: base.commit, from: start.commit, to, commits: [to] }; + if (profile !== 'remote') return { changes }; + const url = (reportedUrl ?? ((head) => ({ open: openOwn(head), report: '' })))(branch); + return { changes: { ...changes, pr_url: url.report || `${url.open}?tab=files#top` } }; +} + +/** Opens a pull request from `head` into main on the host, as a remote driver's own tooling does, and returns its URL. */ +function openOwn(head: string): string { + const file = path.join(ghDir, 'open.json'); + const open = fs.existsSync(file) ? z.array(z.unknown()).parse(JSON.parse(fs.readFileSync(file, 'utf8'))) : []; + const url = `https://github.com/fixture/app/pull/${900 + open.length}`; + const repo = { full_name: 'fixture/app' }; + fs.writeFileSync(file, JSON.stringify([...open, { state: 'open', html_url: url, head: { ref: head, repo }, base: { ref: 'main', repo } }])); + return url; +} + +/** + * Starts the run on alice's install, drives it with `profile`, and does what the scenario's hold says while it holds. + * `polled` says the driver was still driving when a `poll` hold was released. + */ +async function drive(profile: Profile, scenario: DeliveryConformanceScenario): Promise<{ run: string; member: Member; log?: DriverLog; held: boolean; killed: boolean; polled: boolean }> { + for (const marker of ['release', 'lookup.held', 'create.held', 'ship-saw.json']) fs.rmSync(path.join(ghDir, marker), { force: true }); + gh.configure({ ...(scenario.unreachable ? { lookup: 'fail' as const } : {}), ...(scenario.hold ? { hold: scenario.hold.call } : {}) }); + const heldFile = scenario.hold && path.join(ghDir, `${scenario.hold.call}.held`); + const release = (): void => fs.writeFileSync(path.join(ghDir, 'release'), ''); + let run = ''; + let stopped = false; + let finished = false; + let polled = false; + let watcher: Promise | undefined; + // A process of the same machine that polls without taking leases starts the delivery and holds it. + const deliverElsewhere = async (step: string): Promise => { + if (step !== 'work' || scenario.hold?.action !== 'poll' || !heldFile) return; + const poller = { kind: 'watcher', machine: 'alice', workspace: workOf('alice') }; + let exited = false; + watcher = teamai('alice', ['workflow', 'next', run, '--json', JSON.stringify({ driver: poller, canTakeLease: false })]); + void watcher.then(() => { exited = true; }); + await waitFor(() => fs.existsSync(heldFile) || exited, 30_000); + }; + + let driving: Promise; + if (profile === 'local' && scenario.hold?.action === 'poll') { + // The local driver delivers in its own process right after it reports, so here a session driver reports work and + // stops while another process delivers it, and the local driver resumes the run meanwhile. + run = String(envelope(await teamai('alice', ['workflow', 'start', 'deliver'])).data?.run); + await stubDriver('session', 'alice', run, { afterReport: deliverElsewhere, stopped: () => watcher !== undefined }); + driving = teamai('alice', ['workflow', 'resume', run]); + } else if (profile === 'local') { + driving = teamai('alice', ['workflow', 'start', 'deliver', '--driver', 'local'], { stderr: (text) => { + run ||= /Run (r[0-9a-f]{16}) started/.exec(text)?.[1] ?? ''; + } }); + expect(await waitFor(() => run !== '', 30_000), 'the local driver printed its run').toBe(true); + } else { + const started = await teamai('alice', ['workflow', 'start', 'deliver']); + run = String(envelope(started).data?.run); + driving = stubDriver(profile, 'alice', run, { stopped: () => stopped, afterReport: deliverElsewhere }); + } + void driving.then(() => { finished = true; }, () => { finished = true; }); + + const held = heldFile !== undefined && await waitFor(() => fs.existsSync(heldFile) || finished, 60_000) && fs.existsSync(heldFile); + let member: Member = 'alice'; + let log: DriverLog | undefined; + let killed = false; + if (held && scenario.hold && heldFile) { + const pid = Number(fs.readFileSync(heldFile, 'utf8')); + switch (scenario.hold.action) { + case 'crash': + process.kill(pid, 'SIGKILL'); + release(); + break; + case 'publish': { + const published = await teamai('alice', ['workflow', 'publish', run]); + expect(envelope(published), said(published)).toMatchObject({ data: { publication: 'published' } }); + stopped = true; + process.kill(pid, 'SIGKILL'); + release(); + await driving.catch(() => undefined); + member = 'bob'; + break; + } + case 'cancel': { + const cancelled = await teamai('alice', ['workflow', 'cancel', run]); + expect(cancelled.code, said(cancelled)).toBe(0); + release(); + break; + } + case 'poll': + await sleep(HOLD_MS); + polled = !finished; + release(); + break; + default: { + const unknown: never = scenario.hold.action; + throw new Error(`unknown hold action ${String(unknown)}`); + } + } + } + if (member === 'bob') { + if (profile === 'local') { + const resumed = await teamai('bob', ['workflow', 'resume', run]); + expect(envelope(resumed), said(resumed)).toMatchObject({ data: { done: { run } } }); + } else { + log = await stubDriver(profile, 'bob', run, { claim: true }); + } + } else { + const driven = await driving; + if ('leases' in driven) { + log = driven; + killed = driven.killed > 0; + } else if (driven.signal) { + killed = true; + const resumed = await teamai('alice', ['workflow', 'resume', run]); + expect(envelope(resumed), said(resumed)).toMatchObject({ data: { done: { run } } }); + } + } + if (watcher) await watcher; + return { run, member, log, held, killed, polled }; +} + +const PROFILES: Profile[] = ['local', 'session', 'external', 'remote']; + +describe('workflow delivery conformance of each kind of driver through the CLI', () => { + beforeAll(async () => { + sandbox = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-delivery-drivers-'))); + teamOrigin = path.join(sandbox, 'team.git'); + productOrigin = path.join(sandbox, 'app.git'); + ghDir = path.join(sandbox, 'gh'); + git(sandbox, ['init', '-q', '--bare', '-b', 'main', teamOrigin]); + git(sandbox, ['init', '-q', '--bare', '-b', 'main', productOrigin]); + seed(path.join(sandbox, 'seed-team'), { 'teamai.yaml': `team: acme\nrepo: ${TEAM_URL}\nprovider: github\n`, 'workflows/deliver.yaml': deliveryConformanceWorkflow }, teamOrigin); + seed(path.join(sandbox, 'seed-app'), { 'README.md': '# app\n' }, productOrigin); + gh = fakeGhHost(ghDir); + await join('alice'); + await join('bob'); + }, 120_000); + + afterAll(() => { + if (sandbox) fs.rmSync(sandbox, { recursive: true, force: true }); + }); + + for (const scenario of workflowDeliveryConformanceScenarios) { + for (const profile of PROFILES) { + it(`${profile}: ${scenario.name}`, async () => { + const expected = profile === 'remote' ? scenario.reported : scenario.opened; + const { run, member, log, held, killed, polled } = await drive(profile, scenario); + const state = readRun(member, run); + const branch = `teamai/${run}/work/p1`; + const work = state.steps.find((step) => step.id === 'work'); + const steps = state.steps.map((step) => `${step.id} ${step.status}${step.failure ? `: ${step.failure.message}` : ''}`).join('\n'); + + // A driver that reports its own pull request is looked up only when it reports, and never reaches a create. + const holds = profile === 'remote' ? scenario.hold?.call === 'lookup' : scenario.hold !== undefined; + expect(held, 'the case reached the call it holds').toBe(holds); + expect(killed, 'the process inside the create was killed').toBe(holds && scenario.hold?.action === 'crash'); + if (holds && scenario.hold?.action === 'poll') { + expect(polled, 'the driver kept driving while the delivery held').toBe(true); + if (log) expect(log.waits, 'the driver polled while the delivery held').toBeGreaterThan(0); + } + expect([state.status, work?.status], steps).toEqual([expected.run, expected.work]); + expect(createsOf(run)).toEqual(Array.from({ length: expected.creates }, () => ['fixture/app', branch, 'main'])); + expect(state.deliveries ?? []).toHaveLength(expected.prs); + expect(endRecordPrs(run) ?? []).toEqual((state.deliveries ?? []).map((receipt) => receipt.url)); + for (const receipt of state.deliveries ?? []) { + expect(receipt).toMatchObject({ branch, base: 'main', step: 'work', pass: 1 }); + expect(receipt.url).toMatch(/^https:\/\/github\.com\/fixture\/app\/pull\/\d+$/); + } + if (expected.failure) expect(work?.failure).toMatchObject({ code: expected.failure, message: expect.stringContaining(branch) }); + const tip = git(sandbox, ['--git-dir', productOrigin, 'rev-parse', `refs/heads/${branch}`]); + if (work?.changes) expect(tip, 'the branch is on origin').toBe(work.changes.to); + if (expected.run === 'succeeded') { + const saw = OpenSchema.parse(JSON.parse(fs.readFileSync(path.join(ghDir, 'ship-saw.json'), 'utf8'))); + expect(saw.filter((pull) => pull.head.ref === branch).map((pull) => pull.html_url), 'ship ran once work was delivered') + .toEqual((state.deliveries ?? []).map((receipt) => receipt.url)); + } + if (profile === 'external' && log) { + expect(log.reports.map((report) => report.step), 'it reports only what it was leased').toEqual(log.leases); + expect(log.reports.filter((report) => report.step === 'work').map((report) => report.status), 'its report leaves work delivering, for TeamAI to settle') + .toEqual(log.leases.includes('work') ? ['delivering'] : []); + expect(log.waits, 'it sleeps on each wait').toBeLessThanOrEqual(HOLD_MS / POLL_MS + 2); + expect(log.nexts, 'its next calls stay bounded').toBeLessThanOrEqual(log.leases.length + log.waits + 2); + } + }, 120_000); + } + } + + it('remote: a reported URL with credentials is named in the report envelope without its value, and the open pull request of the branch is delivered', async () => { + gh.configure({}); + const started = await teamai('alice', ['workflow', 'start', 'deliver']); + const run = String(envelope(started).data?.run); + let opened = ''; + const reports: Result[] = []; + await stubDriver('remote', 'alice', run, { + reportedUrl: (branch) => { + opened = openOwn(branch); + return { open: opened, report: opened.replace('https://', 'https://x-access-token:ghs_droppedvalue@') }; + }, + onReport: (step, result) => { if (step === 'work') reports.push(result); }, + }); + const [report] = reports; + expect(report && envelope(report)).toMatchObject({ data: { accepted: true, step: { id: 'work', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } }); + expect(report && said(report)).not.toContain('ghs_droppedvalue'); + const state = readRun('alice', run); + expect(state.status).toBe('succeeded'); + expect(state.deliveries?.map((receipt) => receipt.url)).toEqual([opened]); + expect(createsOf(run)).toEqual([]); + expect(JSON.stringify(state)).not.toContain('ghs_droppedvalue'); + }, 120_000); +}); diff --git a/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts new file mode 100644 index 000000000..a8a439a41 --- /dev/null +++ b/src/__tests__/e2e/workflow-pr-delivery-1090.test.ts @@ -0,0 +1,257 @@ +/** + * Real CLI: the "Start from CI" job in docs/guide/workflow.md, run the way GitHub Actions runs it. The test reads the + * job's checkout ref, `env` and `run:` commands from the guide, fills in the Actions expressions, and runs each command + * with bash in a clone of the product, where `npx teamai-cli` is the built CLI and `gh` is the fake every delivery test + * shares. The product's origin is GitHub, or a host TeamAI reaches through git alone, and the team repo is on a generic + * git host; git rewrites each to a bare repository. + */ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { ciRecipesOf, runCommandsOf, type CiRecipe } from '../helpers/ci-recipe.js'; +import { trackDetachedProcesses } from '../helpers/detached-processes.js'; +import { fakeGhHost, type FakeGh } from '../helpers/fake-gh.js'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); +const CLI = path.join(ROOT, 'dist', 'index.js'); +const GUIDE = path.join(ROOT, 'docs', 'guide', 'workflow.md'); +const TEAM_URL = 'https://git.example.internal/acme/team.git'; +const GITHUB_URL = 'https://github.com/fixture/app.git'; +const GENERIC_URL = 'https://git.example.internal/acme/app.git'; +const ISSUE = 42; +/** What each Actions expression in the recipe stands for in the fixture. */ +const EXPRESSIONS: Record = { + 'github.event.issue.number': String(ISSUE), + 'vars.TEAM_REPO': TEAM_URL, + 'secrets.WORKFLOW_TOKEN': 'fixture-token', +}; + +const workflow = (required: boolean): string => `schemaVersion: v1alpha1 +name: ship-issue +description: Write the issue number on a branch and deliver it for review. +inputs: { issue: number } +steps: + - id: approve + approve: { message: Ship it?, required: ${required} } + - id: work + needs: [approve] + workspace: new + deliver: pr + command: node -e "require('fs').writeFileSync('issue.txt', process.env.ISSUE)" + env: + ISSUE: "{{ inputs.issue }}" +`; + +const DoneSchema = z.object({ data: z.object({ done: z.object({ run: z.string(), outcome: z.string() }) }) }); +const RunSchema = z.object({ + steps: z.array(z.object({ + id: z.string(), status: z.string(), + changes: z.object({ to: z.string() }).optional(), + failure: z.object({ code: z.string(), message: z.string() }).optional(), + })), + deliveries: z.array(z.object({ url: z.string(), branch: z.string(), base: z.string(), step: z.string(), pass: z.number() })).optional(), +}); +const RecordSchema = z.object({ run: z.string(), prs: z.array(z.string()).optional() }); +const OpenSchema = z.array(z.object({ html_url: z.string(), head: z.object({ ref: z.string() }) })); + +interface Ran { + command: string; + status: number | null; + stdout: string; + stderr: string; +} + +/** Each fixture root, and a wait for the detached processes its commands left running, which still write there. */ +const fixtures: Array<{ root: string; settled: () => Promise }> = []; + +afterAll(async () => { + for (const { root, settled } of fixtures.splice(0)) { + await settled(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +function git(cwd: string, args: string[], env: NodeJS.ProcessEnv = process.env): string { + const result = spawnSync('git', args, { cwd, env, encoding: 'utf8' }); + if (result.status !== 0) throw new Error(`git ${args.join(' ')} failed:\n${result.stderr}`); + return result.stdout.trim(); +} + +/** `text` with each `${{ }}` replaced as Actions does before the shell runs. */ +function expand(text: string): string { + return text.replace(/\$\{\{\s*(.+?)\s*\}\}/g, (_, expression: string) => { + const value = EXPRESSIONS[expression]; + if (value === undefined) throw new Error(`the recipe uses \${{ ${expression} }}, which the fixture does not fill in`); + return value; + }); +} + +function seed(root: string, bare: string, files: Record): void { + const dir = fs.mkdtempSync(path.join(root, 'seed-')); + git(root, ['init', '-q', '--bare', '-b', 'main', bare]); + git(dir, ['init', '-q', '-b', 'main']); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), text); + } + git(dir, ['add', '-A']); + git(dir, ['-c', 'user.name=Seed', '-c', 'user.email=seed@example.com', 'commit', '-qm', 'seed']); + git(dir, ['push', '-q', bare, 'main']); +} + +/** + * The run state `run` left in the data home of the checkout's workspace scope, `~/.teamai/projects//`, not + * the copy of its record in that data home's reports worktree. + */ +function runState(root: string, run: string): z.infer { + const state = new RegExp(`^home/\\.teamai/projects/[^/]+/workflows/runs/${run}\\.json$`); + const file = fs.readdirSync(root, { recursive: true, encoding: 'utf8' }).find((name) => state.test(name.split(path.sep).join('/'))); + if (!file) throw new Error(`no run state for ${run} under ${root}`); + return RunSchema.parse(JSON.parse(fs.readFileSync(path.join(root, file), 'utf8'))); +} + +/** + * Runs the recipe's one job against a fresh product whose origin is `origin`, with a team repo whose `ship-issue` + * workflow asks an approval that is `required` or optional. A step that fails stops the job, as in Actions. + */ +function runRecipe(origin: string, required: boolean): { recipe: CiRecipe; ran: Ran[]; root: string; gh: FakeGh; team: string; product: string } { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-ci-recipe-'))); + const detached = trackDetachedProcesses(root); + fixtures.push({ root, settled: detached.waitForExit }); + const home = path.join(root, 'home'); + const bin = path.join(root, 'bin'); + const team = path.join(root, 'team.git'); + const product = path.join(root, 'app.git'); + for (const dir of [home, bin, path.join(home, '.gh')]) fs.mkdirSync(dir); + const gh = fakeGhHost(path.join(root, 'gh')); + fs.writeFileSync(path.join(bin, 'npx'), [ + '#!/bin/sh', + '[ "$1" = teamai-cli ] || { echo "the recipe ran npx $*" >&2; exit 97; }', + 'shift', + `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(CLI)} "$@"`, '', + ].join('\n'), { mode: 0o755 }); + fs.writeFileSync(path.join(home, '.gitconfig'), [ + '[user]', '\tname = runner', '\temail = runner@example.com', + `[url "file://${team}"]`, `\tinsteadOf = ${TEAM_URL}`, + `[url "file://${product}"]`, `\tinsteadOf = ${origin}`, '', + ].join('\n')); + seed(root, team, { 'teamai.yaml': `team: acme\nrepo: ${TEAM_URL}\n`, 'workflows/ship-issue.yaml': workflow(required) }); + seed(root, product, { 'README.md': '# app\n' }); + + const env: NodeJS.ProcessEnv = { + ...process.env, + HOME: home, USERPROFILE: home, GIT_CONFIG_GLOBAL: path.join(home, '.gitconfig'), GH_CONFIG_DIR: path.join(home, '.gh'), + PATH: [bin, path.dirname(gh.bin), process.env.PATH ?? ''].join(path.delimiter), + GIT_TERMINAL_PROMPT: '0', GH_PROMPT_DISABLED: '1', FORCE_COLOR: '0', + NODE_OPTIONS: [process.env.NODE_OPTIONS, detached.nodeOptions].filter(Boolean).join(' '), + }; + for (const name of ['XDG_CONFIG_HOME', 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'GITHUB_TOKEN', 'GH_TOKEN']) delete env[name]; + + const [recipe, ...others] = ciRecipesOf(GUIDE); + if (!recipe || others.length > 0) throw new Error(`${GUIDE} shows ${others.length + (recipe ? 1 : 0)} GitHub Actions workflows, not one`); + const [job, ...jobs] = Object.values(recipe.jobs); + if (!job || jobs.length > 0) throw new Error('the recipe has more than one job'); + const checkout = job.steps.find((step) => step.uses?.startsWith('actions/checkout@')); + const ref = z.string().parse(checkout?.with?.ref); + const work = path.join(root, 'work'); + git(root, ['clone', '-q', '-b', ref, origin, work], env); + const jobEnv = Object.fromEntries(Object.entries(job.env ?? {}).map(([name, value]) => [name, expand(value)])); + + const ran: Ran[] = []; + for (const command of runCommandsOf(recipe)) { + const result = spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', expand(command)], { + cwd: work, env: { ...env, ...jobEnv }, encoding: 'utf8', timeout: 120_000, + }); + ran.push({ command, status: result.status, stdout: result.stdout, stderr: result.stderr }); + if (result.status !== 0) break; + } + return { recipe, ran, root, gh, team, product }; +} + +function said(ran: Ran[]): string { + return ran.map((step) => `$ ${step.command}\nexit ${step.status}\n${step.stdout}\n${step.stderr}`).join('\n'); +} + +function doneOf(step: Ran | undefined): { run: string; outcome: string } { + const line = step?.stdout.trim().split('\n').filter((item) => item.startsWith('{')).at(-1); + if (!line) throw new Error(`no JSON envelope from ${step?.command}`); + return DoneSchema.parse(JSON.parse(line)).data.done; +} + +function createsOf(gh: FakeGh): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr' && call.args[1] === 'create') + .map((call) => ['-R', '-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +/** The query the guide's run records section gives for the run that opened `url`, run over the records on `team`. */ +function runThatOpened(root: string, team: string, url: string): string { + const query = /^jq -r .+ workflows\/runs\/\*\.json$/m.exec(fs.readFileSync(GUIDE, 'utf8'))?.[0]; + if (!query) throw new Error(`${GUIDE} shows no jq query over workflows/runs`); + const reports = path.join(root, 'reports'); + fs.mkdirSync(path.join(reports, 'workflows', 'runs'), { recursive: true }); + for (const file of git(root, ['--git-dir', team, 'ls-tree', '--name-only', 'teamai-reports', 'workflows/runs/']).split('\n').filter(Boolean)) { + fs.writeFileSync(path.join(reports, file), git(root, ['--git-dir', team, 'show', `teamai-reports:${file}`])); + } + const result = spawnSync('bash', ['-c', query.replace('', url)], { cwd: reports, encoding: 'utf8' }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); +} + +const hasJq = spawnSync('jq', ['--version']).status === 0; + +describe('the CI recipe in the workflow guide, run through the real CLI', () => { + describe('on a GitHub origin', () => { + let job: ReturnType; + let done: { run: string; outcome: string }; + const url = 'https://github.com/fixture/app/pull/1'; + beforeAll(() => { + job = runRecipe(GITHUB_URL, false); + expect(job.ran.map((step) => step.status), said(job.ran)).toEqual(runCommandsOf(job.recipe).map(() => 0)); + done = doneOf(job.ran.at(-1)); + }, 180_000); + + it('opens one pull request from the delivered branch into the checked-out branch, and lists it in the run record', () => { + const { root, gh, team, product } = job; + expect(done.outcome).toBe('succeeded'); + const branch = `teamai/${done.run}/work/p1`; + expect(createsOf(gh)).toEqual([['fixture/app', branch, 'main']]); + const opened = OpenSchema.parse(JSON.parse(fs.readFileSync(path.join(path.dirname(gh.bin), 'open.json'), 'utf8'))); + expect(opened.map((pull) => [pull.head.ref, pull.html_url])).toEqual([[branch, url]]); + expect(runState(root, done.run).deliveries).toEqual([{ url, branch, base: 'main', step: 'work', pass: 1 }]); + expect(git(root, ['--git-dir', product, 'show', `${branch}:issue.txt`])).toBe(String(ISSUE)); + const record = RecordSchema.parse(JSON.parse(git(root, ['--git-dir', team, 'show', `teamai-reports:workflows/runs/${done.run}.json`]))); + expect(record.prs).toEqual([url]); + }); + + it.skipIf(!hasJq)('finds the run that opened the pull request with the jq query of the guide', () => { + expect(runThatOpened(job.root, job.team, url)).toBe(done.run); + }); + }); + + it('fails a run whose origin TeamAI reaches through git alone, naming the branch it pushed there', () => { + const { recipe, ran, root, gh, product } = runRecipe(GENERIC_URL, false); + + expect(ran.map((step) => step.status), said(ran)).toEqual(runCommandsOf(recipe).map((_, index, all) => (index === all.length - 1 ? 1 : 0))); + const done = doneOf(ran.at(-1)); + expect(done.outcome).toBe('failed'); + const branch = `teamai/${done.run}/work/p1`; + const work = runState(root, done.run).steps.find((step) => step.id === 'work'); + expect(work?.failure).toMatchObject({ code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) }); + expect(git(root, ['--git-dir', product, 'rev-parse', `refs/heads/${branch}`])).toBe(work?.changes?.to); + expect(createsOf(gh)).toEqual([]); + }, 180_000); + + it('fails a required approval unattended with hitl_required, and opens nothing', () => { + const { recipe, ran, root, gh } = runRecipe(GITHUB_URL, true); + + expect(ran.map((step) => step.status), said(ran)).toEqual(runCommandsOf(recipe).map((_, index, all) => (index === all.length - 1 ? 1 : 0))); + const done = doneOf(ran.at(-1)); + expect(done.outcome).toBe('failed'); + expect(runState(root, done.run).steps.find((step) => step.id === 'approve')?.failure).toMatchObject({ code: 'hitl_required' }); + expect(createsOf(gh)).toEqual([]); + }, 180_000); +}); diff --git a/src/__tests__/helpers/ci-recipe.ts b/src/__tests__/helpers/ci-recipe.ts new file mode 100644 index 000000000..3415ab41e --- /dev/null +++ b/src/__tests__/helpers/ci-recipe.ts @@ -0,0 +1,26 @@ +import fs from 'node:fs'; +import { parseDocument } from 'yaml'; +import { z } from 'zod'; + +const RecipeSchema = z.object({ + jobs: z.record(z.object({ + 'runs-on': z.string().optional(), + env: z.record(z.string()).optional(), + steps: z.array(z.object({ uses: z.string().optional(), with: z.record(z.unknown()).optional(), run: z.string().optional() })), + })), +}); +export type CiRecipe = z.infer; + +/** Each GitHub Actions workflow the page at `file` shows: a yaml fence whose document has `jobs`. */ +export function ciRecipesOf(file: string): CiRecipe[] { + const text = fs.readFileSync(file, 'utf8'); + return [...text.matchAll(/^```yaml\n([\s\S]*?)^```/gm)].flatMap((match) => { + const value: unknown = parseDocument(match[1] ?? '').toJS(); + return value && typeof value === 'object' && 'jobs' in value ? [RecipeSchema.parse(value)] : []; + }); +} + +/** The `run:` commands of every job in `recipe`, in order. */ +export function runCommandsOf(recipe: CiRecipe): string[] { + return Object.values(recipe.jobs).flatMap((job) => job.steps.flatMap((step) => (step.run === undefined ? [] : [step.run]))); +} diff --git a/src/__tests__/helpers/fake-gh.ts b/src/__tests__/helpers/fake-gh.ts new file mode 100644 index 000000000..c4431846c --- /dev/null +++ b/src/__tests__/helpers/fake-gh.ts @@ -0,0 +1,103 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { vi } from 'vitest'; +import { z } from 'zod'; +import * as cliPath from '../../utils/cli-path.js'; + +const CallSchema = z.object({ args: z.array(z.string()), cwd: z.string(), prompt: z.string().optional() }); +export type GhCall = z.infer; + +/** + * How the fake answers. `lookup: 'fail'` fails every `gh api`. `create: 'fail'` fails `gh pr create` and opens nothing; + * `'fail-after-open'` opens the request and still fails, as a create that timed out after GitHub took it. `open` + * replaces the open requests. `hold` waits inside that call, a create once it recorded the request, until a file named + * `release` is in the fake's directory, or 20 seconds passed; while it waits, `.held` there holds the pid of the + * process that called it. + */ +export interface FakeGhOptions { + lookup?: 'fail'; + create?: 'fail' | 'fail-after-open'; + open?: unknown[]; + hold?: 'lookup' | 'create'; +} + +function script(dir: string): string { + return `#!${process.execPath} +const fs = require('node:fs'); +const path = require('node:path'); +const dir = ${JSON.stringify(dir)}; +const read = (file, fallback) => { try { return JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')); } catch { return fallback; } }; +const args = process.argv.slice(2); +const options = read('options.json', {}); +fs.appendFileSync(path.join(dir, 'calls.jsonl'), JSON.stringify({ args, cwd: process.cwd(), prompt: process.env.GIT_TERMINAL_PROMPT }) + '\\n'); +const open = read('open.json', []); +const hold = (call) => { + if (options.hold !== call) return; + fs.writeFileSync(path.join(dir, call + '.held'), String(process.ppid)); + for (const until = Date.now() + 20000; !fs.existsSync(path.join(dir, 'release')) && Date.now() < until;) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); +}; +if (args[0] === 'api') { + if (options.lookup === 'fail') { console.error('HTTP 401: Bad credentials for token ghp_fake_secret'); process.exit(1); } + hold('lookup'); + console.log(JSON.stringify(open)); + process.exit(0); +} +if (args[0] === 'pr' && args[1] === 'create') { + const value = (flag) => args[args.indexOf(flag) + 1]; + const repo = value('-R'); + const pull = { state: 'open', html_url: 'https://github.com/' + repo + '/pull/' + (open.length + 1), head: { ref: value('-H'), repo: { full_name: repo } }, base: { ref: value('-B'), repo: { full_name: repo } } }; + if (options.create !== 'fail') fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify([...open, pull])); + hold('create'); + if (options.create) { console.error('pull request create failed for token ghp_fake_secret'); process.exit(1); } + console.log(pull.html_url); + process.exit(0); +} +console.error('unexpected gh call: ' + args.join(' ')); +process.exit(9); +`; +} + +export interface FakeGh { + /** The executable, named `gh`. */ + bin: string; + configure(next: FakeGhOptions): void; + calls(): GhCall[]; +} + +/** + * A `gh` that keeps open pull requests under `dir` and logs each call with its cwd and GIT_TERMINAL_PROMPT. It answers + * `gh api` with every open request and `gh pr create` with a new one, and its failures print a token, so a test can + * show none is kept. teamai resolves `gh` to it until the test's mocks are restored. + */ +export function fakeGh(dir: string, options: FakeGhOptions = {}): FakeGh { + const fake = fakeGhHost(dir, options); + resolveGh(fake.bin); + return fake; +} + +/** The same `gh` for a process this test spawns, which finds it first on its PATH. */ +export function fakeGhHost(dir: string, options: FakeGhOptions = {}): FakeGh { + fs.mkdirSync(dir, { recursive: true }); + const bin = path.join(dir, 'gh'); + fs.writeFileSync(bin, script(dir), { mode: 0o755 }); + const fake = { + bin, + configure(next: FakeGhOptions): void { + fs.writeFileSync(path.join(dir, 'options.json'), JSON.stringify(next)); + if (next.open) fs.writeFileSync(path.join(dir, 'open.json'), JSON.stringify(next.open)); + }, + calls(): GhCall[] { + const log = path.join(dir, 'calls.jsonl'); + if (!fs.existsSync(log)) return []; + return fs.readFileSync(log, 'utf8').trim().split('\n').map((line) => CallSchema.parse(JSON.parse(line))); + }, + }; + fake.configure(options); + return fake; +} + +/** Makes teamai find `gh` at `bin`, or find none, whatever this machine has installed. */ +export function resolveGh(bin: string | null): void { + const resolve = cliPath.resolveCliPath; + vi.spyOn(cliPath, 'resolveCliPath').mockImplementation((cmd, platform) => (cmd === 'gh' ? bin : resolve(cmd, platform))); +} diff --git a/src/__tests__/init.test.ts b/src/__tests__/init.test.ts index 8b440bb4a..1a9525de1 100644 --- a/src/__tests__/init.test.ts +++ b/src/__tests__/init.test.ts @@ -84,6 +84,7 @@ vi.mock('../providers/tgit/gf-cli.js', () => { ensureGfInstalled: () => mockEnsureGfInstalled(), ensureAuthenticated: vi.fn().mockReturnValue('testuser'), isGfInstalled: vi.fn().mockReturnValue(true), + tgitFindOpenMr: vi.fn(), RepoNotFoundError, }; }); diff --git a/src/__tests__/no-provider-merge.test.ts b/src/__tests__/no-provider-merge.test.ts new file mode 100644 index 000000000..fc49eb2d2 --- /dev/null +++ b/src/__tests__/no-provider-merge.test.ts @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, expectTypeOf, it } from 'vitest'; +import type { PullRequestLookup } from '../providers/types.js'; + +const SRC = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +/** Every provider file, a new adapter included, and the delivery code that calls them. */ +function scope(): string[] { + const providers = fs.readdirSync(path.join(SRC, 'providers'), { recursive: true, encoding: 'utf8' }) + .filter((file) => file.endsWith('.ts')) + .map((file) => path.join('providers', file)); + return [...providers, path.join('workflow', 'pr-delivery.ts')]; +} + +const MERGE_CALLS = [ + /['"`](?:pr|mr)['"`]\s*,\s*['"`]merge/i, + /\b(?:pr|mr)\s+merge\b/i, + /['"`]merge(?:[-_]?pulls?)?['"`]/i, + // A REST merge or accept endpoint, not the words "pull/merge request". + /\/merge\b(?!\s+requests?)/i, + /\/accept\b/i, + /auto_?merge|merge_when_pipeline_succeeds|mergePullRequest/i, +]; + +describe('workflow delivery never merges', () => { + it('no provider file or delivery code holds a merge, accept or auto-merge call', () => { + const found = scope().flatMap((file) => { + const text = fs.readFileSync(path.join(SRC, file), 'utf8'); + return MERGE_CALLS.filter((pattern) => pattern.test(text)).map((pattern) => `${file}: ${pattern}`); + }); + expect(found).toEqual([]); + }); + + it('the lookup delivery reads providers through can only find an open request', () => { + expectTypeOf().toEqualTypeOf<'findOpen'>(); + }); + + it('delivery code imports nothing from the engine, which applies its outcome', () => { + expect(fs.readFileSync(path.join(SRC, 'workflow', 'pr-delivery.ts'), 'utf8')).not.toMatch(/from '\.\/engine\.js'/); + }); + + it('the run snapshot loads neither delivery code nor a provider', () => { + const loaded = [...runtimeImports(path.join('workflow', 'snapshot.ts'))]; + expect(loaded.filter((file) => file.startsWith(`providers${path.sep}`) || file === path.join('workflow', 'pr-delivery.ts'))).toEqual([]); + expect(loaded).toContain(path.join('workflow', 'delivery-schema.ts')); + }); +}); + +/** `file` and every module it loads, following each relative import or re-export that is not `import type`. */ +function runtimeImports(file: string, seen = new Set()): Set { + if (seen.has(file)) return seen; + seen.add(file); + const text = fs.readFileSync(path.join(SRC, file), 'utf8'); + for (const [, specifier] of text.matchAll(/^(?:import|export)(?!\s+type\b)[^;]*?\sfrom\s+'(\.[^']*)\.js'/gm)) { + runtimeImports(path.join(path.dirname(file), `${specifier}.ts`), seen); + } + return seen; +} diff --git a/src/__tests__/workflow-delivery.test.ts b/src/__tests__/workflow-delivery.test.ts new file mode 100644 index 000000000..e3f1c37ee --- /dev/null +++ b/src/__tests__/workflow-delivery.test.ts @@ -0,0 +1,1869 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fc from 'fast-check'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { z } from 'zod'; +import { parseWorkflow } from '../workflow/parse.js'; +import { leaseNext, parseReport, reportAttempt, startRun, withIgnored, type NextOptions } from '../workflow/engine.js'; +import { knownSecrets } from '../env-resolution.js'; +import { sanitizePrUrl } from '../workflow/pr-url.js'; +import { syncWorkflowDefinitions } from '../workflow/cache.js'; +import { deliveryLockPath, readRun, runLockPath } from '../workflow/store.js'; +import * as store from '../workflow/store.js'; +import { _setDriverPollMs, runLocalDriver } from '../workflow/local-driver.js'; +import { acquireLock, releaseLock } from '../update.js'; +import { AGENT_PATHS, DELIVERY_ONLY_PATHS, protectedPaths } from '../workflow/worktree.js'; +import { fakeGh, type FakeGhOptions } from './helpers/fake-gh.js'; +import { deliverPr } from '../workflow/pr-delivery.js'; +import { buildEndRecord, pendingRecords } from '../workflow/records.js'; +import { projectRun } from '../workflow/snapshot.js'; +import * as worktree from '../workflow/worktree.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import * as registry from '../providers/registry.js'; +import { GitHubProvider } from '../providers/github/index.js'; +import type { GitProvider, PullRequestLookup } from '../providers/types.js'; +import type { LocalConfig } from '../types.js'; +import type { RunState } from '../workflow/types.js'; + +_setDriverPollMs(20); + +const dirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function git(cwd: string, args: string[]): string { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] }).trim(); +} + +async function fixture(steps: string, inputs?: unknown): Promise<{ config: LocalConfig; cwd: string; runId: string; root: string }> { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-delivery-')); + dirs.push(root); + const cwd = path.join(root, 'product'); + const knowledge = path.join(root, 'team'); + fs.mkdirSync(cwd); + fs.mkdirSync(path.join(knowledge, 'workflows'), { recursive: true }); + git(root, ['init', '-q', '--bare', 'remote.git']); + git(cwd, ['init', '-q', '-b', 'main']); + git(cwd, ['config', 'user.name', 'Test']); + git(cwd, ['config', 'user.email', 'test@example.com']); + fs.writeFileSync(path.join(cwd, '.gitignore'), '.teamai/\n'); + fs.writeFileSync(path.join(cwd, 'base.txt'), 'base\n'); + git(cwd, ['add', '-A']); git(cwd, ['commit', '-qm', 'base']); + git(cwd, ['remote', 'add', 'origin', 'https://github.com/fixture/app.git']); + git(cwd, ['config', `url.${path.join(root, 'remote.git')}.insteadOf`, 'https://github.com/fixture/app.git']); + git(cwd, ['push', '-qu', 'origin', 'main']); + const config: LocalConfig = { repo: { localPath: knowledge, remote: 'local', kind: 'self' }, scope: 'project', + username: 'tester', usageReportEnabled: false, additionalRoles: [], projectRoot: cwd, dataHome: path.join(root, 'data'), workflows: { tools: {}, max_parallel: 1 } }; + fs.writeFileSync(path.join(knowledge, 'workflows', 'ship.yaml'), `schemaVersion: v1alpha1\nname: ship\ndescription: Deliver work\n${inputs ? 'inputs:\n packages: [{ name: string, id: string, after: [string] }]\n' : ''}steps:\n${steps}`); + await syncWorkflowDefinitions(config); + const started = await startRun(config, 'ship', inputs ?? {}, { checkout: cwd, base: { ref: 'main', commit: git(cwd, ['rev-parse', 'HEAD']) }, secrets: await knownSecrets(config) }); + if ('error' in started) throw new Error(started.error.message); + return { config, cwd, root, runId: String(started.data.run) }; +} + +async function state(config: LocalConfig, runId: string): Promise { + const run = await readRun(config, runId); + if (typeof run === 'string') throw new Error(run); + return run; +} + +const LeaseSchema = z.object({ + step: z.string(), attempt: z.string(), revision: z.number(), pass: z.number(), + workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }), start: z.object({ commit: z.string() }).optional() }).optional(), +}); +type Lease = z.infer; +const remote = { kind: 'remote', machine: 'paseo', workspace: 'session-1' }; +const writable: NextOptions = { can: { workspace: ['read-only', 'new', 'continue'] } }; +const head = 'c'.repeat(40); + +/** Takes the next lease as a driver that speaks only CLI JSON does. */ +async function take(config: LocalConfig, runId: string): Promise { + const next = await leaseNext(config, runId, remote, writable); + if (!('data' in next) || !next.data.lease) throw new Error(`no lease: ${JSON.stringify(next)}`); + return LeaseSchema.parse(next.data.lease); +} + +/** Reports through the wire payload `teamai workflow report --json` reads. */ +async function report(config: LocalConfig, runId: string, body: Record) { + const parsed = parseReport({ driver: remote, ...body }); + if (typeof parsed === 'string') throw new Error(parsed); + return withIgnored(await reportAttempt(config, runId, parsed.report), parsed.ignored); +} + +/** Commits `files` on the lease's branch in a clone of its own and pushes it, as a remote driver does, and returns the commit. */ +function pushWork(root: string, lease: Lease, files: Record = { 'work.txt': 'work\n' }): string { + if (!lease.workspace) throw new Error(`lease ${lease.step} has no workspace`); + const clone = fs.mkdtempSync(path.join(root, 'driver-')); + git(root, ['clone', '-q', path.join(root, 'remote.git'), clone]); + git(clone, ['checkout', '-q', '-b', lease.workspace.branch, lease.workspace.base.commit]); + return commitAndPush(clone, lease.workspace.branch, files); +} + +/** Commits `files` on top of `branch` as origin holds it, in a clone of its own, and pushes it, as a driver that went on working does. */ +function pushOnto(root: string, branch: string, files: Record): string { + const clone = fs.mkdtempSync(path.join(root, 'driver-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + return commitAndPush(clone, branch, files); +} + +function commitAndPush(clone: string, branch: string, files: Record): string { + for (const [file, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(clone, file)), { recursive: true }); + fs.writeFileSync(path.join(clone, file), content); + } + git(clone, ['add', '-A']); + git(clone, ['-c', 'user.name=Driver', '-c', 'user.email=driver@example.com', 'commit', '-qm', 'work']); + git(clone, ['push', '-q', 'origin', branch]); + return git(clone, ['rev-parse', 'HEAD']); +} + +/** Reports a lease's success with one commit on its branch (`to`, a commit no repository has unless given), and `changes` fields the driver adds. */ +async function succeed(config: LocalConfig, runId: string, lease: Lease, changes: Record = {}) { + const base = lease.workspace?.base.commit; + const to = typeof changes.to === 'string' ? changes.to : head; + return report(config, runId, { attempt: lease.attempt, revision: lease.revision, outcome: 'success', + ...(lease.workspace ? { changes: { branch: lease.workspace.branch, base, from: base, to, commits: [to], ...changes } } : {}) }); +} + +/** The value that follows `flag` in a gh call. */ +function flag(args: string[], name: string): string | undefined { + const at = args.indexOf(name); + return at < 0 ? undefined : args[at + 1]; +} + +/** An open pull request as GitHub lists it. */ +function pull(number: number, head: string, base = 'main', repo = 'fixture/app') { + return { state: 'open', html_url: `https://github.com/${repo}/pull/${number}`, head: { ref: head, repo: { full_name: repo } }, base: { ref: base, repo: { full_name: repo } } }; +} + +const FIX_THEN_REVIEW = ` - id: fix\n workspace: new\n command: node fix.js\n deliver: pr\n - id: review\n needs: [fix]\n command: echo review\n`; + +/** Two independent steps that continue one shared branch, the first delivering it. */ +const IMPL_AND_DOCS = ` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`; + +describe('workflow PR delivery contract', () => { + it('parses delivery on a writable step and rejects unsupported delivery', () => { + const yaml = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: write\n command: node write.js\n workspace: new\n deliver: pr\n'; + const parsed = parseWorkflow('ship.yaml', yaml); + expect(parsed.ok && parsed.definition.steps[0]).toMatchObject({ deliver: 'pr' }); + expect(parseWorkflow('ship.yaml', yaml.replace('deliver: pr', 'deliver: merge')).ok).toBe(false); + expect(parseWorkflow('ship.yaml', yaml.replace(' workspace: new\n', '')).ok).toBe(false); + const inherited = 'schemaVersion: v1alpha1\nname: ship\ndescription: Ship changes\nsteps:\n - id: change\n workspace: new\n workflow:\n steps:\n - id: write\n prompt: Edit the code.\n deliver: pr\n'; + expect(parseWorkflow('ship.yaml', inherited).ok).toBe(true); + expect(parseWorkflow('ship.yaml', inherited.replace('prompt: Edit the code.', 'command: echo edit')).ok).toBe(false); + }); + + it('reads the reported URL from changes.pr_url and names unread nested fields without values', () => { + const parsed = parseReport({ attempt: 'a1', revision: 1, outcome: 'success', + driver: { kind: 'remote', machine: 'host', workspace: 'work' }, + changes: { branch: 'work', base: 'base', from: 'base', to: 'head', commits: [], pr_url: 'https://review.example/1', private: 'secret' } }); + expect(parsed).toMatchObject({ report: { prUrl: 'https://review.example/1' }, ignored: ['changes.private'] }); + expect(JSON.stringify(parsed)).not.toContain('secret'); + }); + + it('refuses a reported pull request without branch facts', () => { + const driver = { kind: 'remote', machine: 'host', workspace: 'work' }; + expect(parseReport({ attempt: 'a1', revision: 1, outcome: 'success', driver, changes: { pr_url: 'https://review.example/1' } })) + .toBe('changes must include branch, base, from, and to'); + }); + + it.each([ + ['https://github.com/acme/app/pull/101', { url: 'https://github.com/acme/app/pull/101' }], + ['https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview', { url: 'https://bitbucket.acme.internal/projects/APP/repos/web/pull-requests/7/overview' }], + ['https://review.acme.internal/changes/4821?session=abc#c3', { url: 'https://review.acme.internal/changes/4821' }], + ['https://user:token@github.com/acme/app/pull/1', { reason: 'contains credentials' }], + ['http://github.com/acme/app/pull/1', { reason: 'must use HTTPS' }], + ['not a url', { reason: 'is not a valid URL' }], + [`https://example.com/${'a'.repeat(512)}`, { reason: 'exceeds 512 characters' }], + [101, { reason: 'must be a URL string' }], + ])('sanitizes %s', (value, expected) => expect(sanitizePrUrl(value)).toEqual(expected)); + + const urls = fc.webUrl({ validSchemes: ['https', 'http'], withQueryParameters: true, withFragments: true, authoritySettings: { withUserInfo: true } }); + + it('never keeps credentials, a query or a fragment, gives back what it kept unchanged, and stays within 512 characters', () => { + fc.assert(fc.property(fc.oneof(urls, fc.string(), fc.string({ minLength: 513, maxLength: 700 })), (value) => { + const result = sanitizePrUrl(value); + if (result.url === undefined) return; + const url = new URL(result.url); + expect(url.protocol).toBe('https:'); + expect([url.username, url.password, url.search, url.hash]).toEqual(['', '', '', '']); + expect(result.url.length).toBeLessThanOrEqual(512); + expect(sanitizePrUrl(result.url)).toEqual(result); + }), { numRuns: 500 }); + }); + + it('drops every URL with credentials, and keeps the host and path of any other https URL', () => { + fc.assert(fc.property(urls, (value) => { + const url = new URL(value); + const expected = url.username || url.password ? undefined + : url.protocol === 'https:' && `${url.origin}${url.pathname}`.length <= 512 ? `${url.origin}${url.pathname}` : undefined; + expect(sanitizePrUrl(value).url).toBe(expected); + }), { numRuns: 500 }); + }); +}); + +describe('delivering a step', () => { + it('delivers a driver-reported URL without opening a PR, and leases the dependent in the same next', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + expect(fix).toMatchObject({ step: 'fix', workspace: { branch } }); + + expect(await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://review.acme.internal/changes/4821?session=abc#c3' })) + .toEqual({ data: { accepted: true, revision: 2, step: { id: 'fix', status: 'delivering', outcome: 'success' }, cleanup: true } }); + + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + try { + expect(await leaseNext(config, runId, remote, writable)).toEqual({ data: { wait: { reason: 'fix is delivering', delivering: ['fix'] }, cancelled: [] } }); + } finally { + await releaseLock(held); + } + + const review = await take(config, runId); + expect(review.step).toBe('review'); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', settledAt: 2 }); + expect(run.deliveries).toEqual([{ url: 'https://review.acme.internal/changes/4821', branch, base: 'main', step: 'fix', pass: 1 }]); + expect(gh.calls()).toEqual([]); + + await report(config, runId, { attempt: review.attempt, revision: review.revision, outcome: 'success' }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://review.acme.internal/changes/4821']); + }); + + it('keeps the local driver polling while another process delivers, rather than giving up as busy', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://review.acme.internal/changes/1' }); + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + const released = new Promise((resolve) => setTimeout(() => void releaseLock(held).then(resolve), 200)); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + await released; + }); + + it.each([ + ['a filesystem path', (root: string) => path.join(root, 'remote.git')], + ['a host only git reaches', () => 'https://git.example.internal/fixture/app.git'], + ])('fails a delivery that has no URL and an origin at %s, naming the branch that keeps the work', async (_case, origin) => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + git(cwd, ['remote', 'set-url', 'origin', origin(root)]); + git(cwd, ['config', `url.${path.join(root, 'remote.git')}.insteadOf`, 'https://git.example.internal/fixture/app.git']); + const fix = await take(config, runId); + const to = pushWork(root, fix); + expect(await succeed(config, runId, fix, { to })).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const branch = `teamai/${runId}/fix/p1`; + expect((await state(config, runId)).steps).toMatchObject([ + { id: 'fix', status: 'failed', delivery: { branch, head: to }, failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }, + { id: 'review', status: 'skipped' }, + ]); + expect((await state(config, runId)).steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(gh.calls()).toEqual([]); + }); + + it('opens one pull request from the pushed branch into the run base branch, outside the member checkout, and records it', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + expect(await succeed(config, runId, fix, { to })).toMatchObject({ data: { step: { id: 'fix', status: 'delivering' } } }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries).toEqual([{ url: 'https://github.com/fixture/app/pull/1', branch, base: 'main', step: 'fix', pass: 1 }]); + const [lookup, create, again, ...rest] = gh.calls(); + expect(again?.args).toEqual(lookup?.args); + expect(rest).toEqual([]); + expect(lookup?.args[0]).toBe('api'); + expect(lookup?.prompt).toBe('0'); + const query = new URLSearchParams(lookup?.args[1]?.split('?')[1]); + expect([lookup?.args[1]?.split('?')[0], query.get('state'), query.get('head'), query.get('base')]).toEqual(['repos/fixture/app/pulls', 'open', `fixture:${branch}`, 'main']); + expect(create?.args.slice(0, 2)).toEqual(['pr', 'create']); + expect([flag(create?.args ?? [], '-R'), flag(create?.args ?? [], '-H'), flag(create?.args ?? [], '-B')]).toEqual(['fixture/app', branch, 'main']); + expect(fs.realpathSync(create?.cwd ?? cwd).startsWith(fs.realpathSync(root))).toBe(false); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + }); + + it.each([ + ['a delivery that stopped after opening it', {}], + ['a create that failed after GitHub took it', { create: 'fail-after-open' as const }], + ])('records the pull request %s rather than opening another', async (_case, options) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { ...options, open: 'create' in options ? [] : [pull(5, `${branch}-other`), pull(6, branch, 'main', 'fork/app'), pull(7, branch)] }); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const url = 'create' in options ? 'https://github.com/fixture/app/pull/1' : 'https://github.com/fixture/app/pull/7'; + const run = await state(config, runId); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(gh.calls().map((call) => call.args[0])).toEqual('create' in options ? ['api', 'pr', 'api'] : ['api']); + // The receipt settles the request the run listed before it asked GitHub to open one. + expect(run.creating).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ delivery: { branch } }); + const handed = projectRun(run, {}); + expect(handed.ok && handed.value.steps[0]).toEqual(run.steps[0]); + }); + + it('fails, opening none, when the lookup names an open pull request on another host than the origin', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { open: [{ ...pull(7, branch), html_url: 'https://github.example.net/fixture/app/pull/7' }] }); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.deliveries).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.`) } }); + expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + }); + + it('keeps the work on origin when GitHub cannot be reached, stores none of its output, and retry delivers without running the step again', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`The work is kept on origin in branch ${branch}.`) } }); + expect(failed.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(JSON.stringify(failed)).not.toContain('ghp_fake_secret'); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/${branch}`])).toBe(to); + + gh.configure({}); + expect(await retryStep(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: {} }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('asks the run before it opens a pull request, and opens none for a step that stopped delivering', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + if (!fix.workspace) throw new Error('fix has no workspace'); + const to = pushWork(root, fix); + const job = { key: { step: 'fix', pass: 1, settledAt: 2, branch: fix.workspace.branch, head: to }, runId, runName: 'ship', checkout: cwd, base: fix.workspace.base.commit, target: 'main', childWork: [] }; + + expect(await deliverPr(job, async () => 'refused')).toEqual({ ok: false, fault: { kind: 'withdrawn' } }); + expect(gh.calls().map((call) => call.args[0])).toEqual(['api']); + }); + + it('cancels a delivering step: it delivers nothing and its dependents do not run', async () => { + const { config, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { pr_url: 'https://review.acme.internal/changes/1' }); + + expect(await cancelRun(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'], attempts: [] }, run: { status: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'cancelled', attempt: fix.attempt, delivery: { branch: `teamai/${runId}/fix/p1`, head } }, { id: 'review', status: 'skipped' }, + ]); + expect(run.deliveries).toBeUndefined(); + }); + + it.each([ + ['TeamAI would open the pull request', {}], + ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], + ])('refuses a branch that changes protected paths, naming them, before any pull request is opened or accepted, when %s', async (_case, changes) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const gh = fakeGh(path.join(root, 'gh'), { open: [pull(1, `teamai/${runId}/fix/p1`)] }); + const to = pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n', 'src/app.ts': 'ok\n', 'packages/web/CLAUDE.md': 'rules\n' }); + await succeed(config, runId, fix, { to, ...changes }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('.github/workflows/ci.yml, packages/web/CLAUDE.md.') } }, + { id: 'review', status: 'skipped' }, + ]); + expect(run.steps[0]).not.toMatchObject({ failure: { message: expect.stringContaining('src/app.ts') } }); + if ('pr_url' in changes) expect(run.steps[0]).toMatchObject({ delivery: { reported: changes.pr_url } }); + expect(run.deliveries).toBeUndefined(); + // The report-time lookup only reads; delivery refuses the protected paths before it looks up or opens anything. + expect(gh.calls().map((call) => call.args[0])).toEqual('pr_url' in changes ? ['api'] : []); + }); + + it.each([ + ['TeamAI would open the pull request', {}], + ['the driver reported one', { pr_url: 'https://github.com/fixture/app/pull/1' }], + ])('opens and accepts nothing when origin\'s branch moved past the checked commit to one that changes protected paths, when %s', async (_case, changes) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), { open: 'pr_url' in changes ? [pull(1, branch)] : [] }); + const checked = pushWork(root, fix, { 'src/app.ts': 'ok\n' }); + const ahead = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + expect(await succeed(config, runId, fix, { to: checked, ...changes })).not.toHaveProperty('data.dropped'); + + // No step that writes the branch is leased, so no report can record that commit, and the first call fails the step. + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps).toMatchObject([ + { id: 'fix', status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`branch ${branch} on origin is at commit ${ahead}, which holds commit ${checked}, the last one a report recorded on it, and later commits no report recorded, so TeamAI has not checked them`) } }, + { id: 'review', status: 'skipped' }, + ]); + expect(run.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(run.deliveries).toBeUndefined(); + expect(creates(gh)).toEqual([]); + }); + + it('names a missing origin branch and reads it without credential prompts', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + const branch = fix.workspace?.branch ?? ''; + git(cwd, ['fetch', 'origin', branch]); + git(root, ['--git-dir=remote.git', 'update-ref', '-d', `refs/heads/${branch}`]); + await succeed(config, runId, fix, { to }); + vi.stubEnv('GIT_TERMINAL_PROMPT', '1'); + vi.stubEnv('GIT_ASKPASS', '/opt/editor/askpass.sh'); + const reads = vi.spyOn(worktree, 'git'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`origin has no branch ${branch}; push ${to} to it, then run: teamai workflow retry ${runId} fix`) } }); + expect(reads).toHaveBeenCalledWith(cwd, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', `refs/heads/${branch}`], { timeoutMs: 30000, env: { GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo', GCM_INTERACTIVE: 'never' } }); + expect(run.deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + + it('fetches the commits it checks within the delivery timeout and without credential prompts', async () => { + const { config, cwd, root, runId } = await fixture(FIX_THEN_REVIEW); + fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + // Every git process that reaches origin logs the askpass it was given, then serves the fixture's bare repository. + const [log, transport, bare] = [path.join(root, 'transport.log'), path.join(root, 'transport.sh'), path.join(root, 'remote.git')]; + fs.writeFileSync(transport, `#!/bin/sh\nprintf '%s\\n' "$GIT_ASKPASS" >> '${log}'\nexec git upload-pack "$1"\n`, { mode: 0o755 }); + git(cwd, ['config', '--unset', `url.${bare}.insteadOf`]); + git(cwd, ['config', `url.ext::${transport} ${bare}.insteadOf`, 'https://github.com/fixture/app.git']); + git(cwd, ['config', 'protocol.ext.allow', 'always']); + vi.stubEnv('GIT_ASKPASS', '/opt/editor/askpass.sh'); + const fetches = vi.spyOn(worktree, 'fetchMissingCommits'); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const env = { GIT_TERMINAL_PROMPT: '0', GIT_ASKPASS: 'echo', GCM_INTERACTIVE: 'never' }; + expect(fetches).toHaveBeenCalledWith(cwd, { run: runId, commits: [fix.workspace?.base.commit, to] }, { timeoutMs: 30000, env }); + const askpass = fs.readFileSync(log, 'utf8').trim().split('\n'); + expect(askpass.length).toBeGreaterThanOrEqual(3); + expect(askpass.filter((given) => given !== 'echo')).toEqual([]); + }); + + it('reads a reported head only as a full commit id, so a ref name cannot stand in for the pushed branch', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n' }); + await succeed(config, runId, fix, { to: 'main', pr_url: 'https://github.com/fixture/app/pull/1' }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ + status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`Push branch teamai/${runId}/fix/p1 to origin`) }, + }); + }); + + it('refuses protected changes the branch inherited from a step it needs, though the delivering step left them alone', async () => { + const write = (file: string) => `node -e "require('fs').mkdirSync(require('path').dirname('${file}'),{recursive:true});require('fs').writeFileSync('${file}','x')"`; + const { config, cwd, root, runId } = await fixture(` - id: earlier\n workspace: new\n command: ${write('.github/workflows/ci.yml')}\n - id: later\n needs: [earlier]\n workspace: new\n command: ${write('change.txt')}\n deliver: pr\n`); + + const gh = fakeGh(path.join(root, 'gh')); + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ id: 'later', status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('.github/workflows/ci.yml') } }); + expect(gh.calls()).toEqual([]); + }); + + it('records a pull request once, however many deliveries report it', async () => { + const { config, root, runId } = await fixture(` - id: a\n workspace: new\n command: node a.js\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: node b.js\n deliver: pr\n`); + const url = 'https://review.acme.internal/changes/9'; + const a = await take(config, runId); + await succeed(config, runId, a, { to: pushWork(root, a), pr_url: url }); + const b = await take(config, runId); + await succeed(config, runId, b, { to: pushWork(root, b), pr_url: url }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).deliveries).toEqual([{ url, branch: `teamai/${runId}/a/p1`, base: 'main', step: 'a', pass: 1 }]); + }); + + it('records a pull request delivered after the run ended, and queues the end record again with it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const url = 'https://review.acme.internal/changes/3'; + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: url }); + const read = worktree.fetchMissingCommits; + vi.spyOn(worktree, 'fetchMissingCommits').mockImplementation(async (dir, options) => { + await cancelRun(config, runId, await knownSecrets(config)); + return read(dir, options); + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch: `teamai/${runId}/fix/p1`, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + }); + + it('hands over a delivering step with what it delivers, and the run with its receipts', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const url = 'https://review.acme.internal/changes/5'; + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to, pr_url: url }); + const branch = `teamai/${runId}/fix/p1`; + + const delivering = projectRun(await state(config, runId), {}); + expect(delivering.ok && delivering.value.steps[0]).toMatchObject({ status: 'delivering', delivery: { branch, head: to, reported: url } }); + await take(config, runId); + const delivered = projectRun(await state(config, runId), {}); + expect(delivered.ok && delivered.value.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + }); + +}); + +/** A command that adds a line no earlier run of it wrote to `file`, so every run commits. */ +const bump = (file: string) => `node -e "require('fs').appendFileSync('${file}', Date.now() + ' ' + Math.random() + '\\n')"`; + +/** The head and target branch of each pull request the fake gh was asked to open, in order. */ +function creates(gh: ReturnType): Array<[string | undefined, string | undefined]> { + return gh.calls().filter((call) => call.args[0] === 'pr').map((call) => [flag(call.args, '-H'), flag(call.args, '-B')]); +} + +/** The commits on `head` that `target` does not have, on the fixture's origin, leaving out the merge that brings a copy up to its needs. */ +function commitsBeyond(root: string, target: string, head: string): string[] { + return git(root, ['--git-dir=remote.git', 'log', '--no-merges', '--format=%H', `${target}..${head}`]).split('\n').filter(Boolean); +} + +function changedTo(run: RunState, id: string): string | undefined { + const step = run.steps.find((item) => item.id === id); + return step && 'changes' in step ? step.changes?.to : undefined; +} + +describe('where a pull request goes', () => { + it('opens the pull request of a step into the delivered branch of the step it needs, with only its own commits', async () => { + const { config, cwd, root, runId } = await fixture(` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n deliver: pr\n`); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [a, b] = [`teamai/${runId}/a/p1`, `teamai/${runId}/b/p1`]; + expect(creates(gh)).toEqual([[a, 'main'], [b, a]]); + const run = await state(config, runId); + expect(commitsBeyond(root, a, b)).toEqual([changedTo(run, 'b')]); + expect(run.deliveries).toEqual([ + { url: 'https://github.com/fixture/app/pull/1', branch: a, base: 'main', step: 'a', pass: 1 }, + { url: 'https://github.com/fixture/app/pull/2', branch: b, base: a, step: 'b', pass: 1 }, + ]); + }); + + it('keeps a URL a stacked step reports when it is the open pull request into the delivered branch of the step it needs', async () => { + const { config, root, runId } = await fixture(` - id: a\n workspace: new\n command: node a.js\n deliver: pr\n - id: b\n needs: [a]\n workspace: new\n command: node b.js\n deliver: pr\n`); + const gh = fakeGh(path.join(root, 'gh')); + const [a, b] = [`teamai/${runId}/a/p1`, `teamai/${runId}/b/p1`]; + const first = await take(config, runId); + await succeed(config, runId, first, { to: pushWork(root, first) }); + const second = await take(config, runId); + gh.configure({ open: [pull(1, a), pull(2, b, a)] }); + + const accepted = await succeed(config, runId, second, { to: pushWork(root, second), pr_url: 'https://github.com/fixture/app/pull/2' }); + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'b', status: 'delivering' } } }); + expect(accepted).not.toHaveProperty('data.dropped'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).deliveries?.[1]).toEqual({ url: 'https://github.com/fixture/app/pull/2', branch: b, base: a, step: 'b', pass: 1 }); + expect(creates(gh)).toEqual([[a, 'main']]); + }); + + it('opens the pull request of a for_each item into the delivered branch of the item it runs after, with only its own commits', async () => { + const { config, cwd, root, runId } = await fixture(` - id: build\n for_each: inputs.packages\n workspace: new\n command: ${bump('items.txt')}\n deliver: pr\n`, + { packages: [{ name: 'one', id: 'one', after: [] }, { name: 'two', id: 'two', after: ['one'] }] }); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [one, two] = [`teamai/${runId}/build-0/p1`, `teamai/${runId}/build-1/p1`]; + expect(creates(gh)).toEqual([[one, 'main'], [two, one]]); + expect(commitsBeyond(root, one, two)).toEqual([changedTo(await state(config, runId), 'build[1]')]); + }); + + it('passes over a need that was skipped, to the delivered branch before it', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: retry\n needs: [a]\n if: steps.a.outcome == 'failed'\n workspace: new\n command: ${bump('retry.txt')}\n deliver: pr\n`, + ` - id: c\n needs: [retry, a]\n workspace: new\n command: ${bump('c.txt')}\n deliver: pr\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect((await state(config, runId)).steps[1]).toMatchObject({ id: 'retry', status: 'skipped' }); + expect(creates(gh)).toEqual([[`teamai/${runId}/a/p1`, 'main'], [`teamai/${runId}/c/p1`, `teamai/${runId}/a/p1`]]); + }); + + it('opens the pull request of a sub-workflow\'s first step into the delivered branch of what the workflow step needs', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: change\n needs: [a]\n workflow:\n steps:\n - id: write\n workspace: new\n command: ${bump('write.txt')}\n deliver: pr\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/a/p1`, 'main'], [`teamai/${runId}/change-write/p1`, `teamai/${runId}/a/p1`]]); + }); + + it('opens one pull request for a workflow step with workspace: new, from the branch its steps share, and none for its steps', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: a\n workspace: new\n command: ${bump('a.txt')}\n deliver: pr\n`, + ` - id: change\n needs: [a]\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: write\n workspace: continue\n command: ${bump('write.txt')}\n`, + ` - id: more\n needs: [write]\n workspace: continue\n command: ${bump('more.txt')}\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const [a, change] = [`teamai/${runId}/a/p1`, `teamai/${runId}/change/p1`]; + expect(creates(gh)).toEqual([[a, 'main'], [change, a]]); + const run = await state(config, runId); + expect(commitsBeyond(root, a, change)).toEqual([changedTo(run, 'change.more'), changedTo(run, 'change.write')]); + expect(run.deliveries?.map((receipt) => [receipt.step, receipt.branch])).toEqual([['a', a], ['change', change]]); + }); + + /** A workflow step that delivers its copy, where `a` writes the copy and `b` writes a branch of its own after it. */ + const COPY_AND_SIDE_BRANCH = [ + ` - id: work\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: a\n workspace: continue\n command: ${bump('a.txt')}\n`, + ` - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n`, + ].join(''); + + it('opens no pull request for a workflow step whose branch leaves out the work of a step inside it on another branch', async () => { + const { config, cwd, root, runId } = await fixture(COPY_AND_SIDE_BRANCH); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const [work, b] = [`teamai/${runId}/work/p1`, `teamai/${runId}/work-b/p1`]; + expect(run.steps.find((step) => step.id === 'work.b')).toMatchObject({ status: 'succeeded', changes: { branch: b } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`step "work.b" wrote branch ${b}, which branch ${work} does not hold, so a pull request from it would leave that work out`) } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ failure: { message: expect.stringContaining('Have a later step in the workflow need step "work.b", so it merges branch') } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + }); + + it('fails a workflow step whose own branch holds nothing beyond the run base while a step inside it wrote another branch', async () => { + const { config, cwd, root, runId } = await fixture([ + ` - id: work\n workspace: new\n deliver: pr\n workflow:\n steps:\n`, + ` - id: a\n workspace: continue\n command: node -e ""\n`, + ` - id: b\n needs: [a]\n workspace: new\n command: ${bump('b.txt')}\n`, + ].join('')); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const [work, b] = [`teamai/${runId}/work/p1`, `teamai/${runId}/work-b/p1`]; + expect(run.steps.find((step) => step.id === 'work.a')).toMatchObject({ status: 'succeeded', changes: { branch: work, to: run.base?.commit } }); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`step "work.b" wrote branch ${b}, which branch ${work} does not hold`) } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + }); + + it('opens the pull request of a workflow step once a later step inside it merges the branch another step wrote', async () => { + const { config, cwd, root, runId } = await fixture(`${COPY_AND_SIDE_BRANCH} - id: c\n needs: [b]\n workspace: continue\n command: ${bump('c.txt')}\n`); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const work = `teamai/${runId}/work/p1`; + expect(creates(gh)).toEqual([[work, 'main']]); + expect(commitsBeyond(root, 'main', work)).toContain(changedTo(await state(config, runId), 'work.b')); + }); + + it('opens a pull request for each pass of a looped new step, leaves the earlier ones open, and lists them in order', async () => { + const { config, cwd, root, runId } = await fixture( + ` - id: fix\n workspace: new\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 2", max_iterations: 2 }\n`, + ); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/fix/p1`, 'main'], [`teamai/${runId}/fix/p2`, 'main']]); + expect(gh.calls().every((call) => call.args[0] === 'api' || (call.args[0] === 'pr' && call.args[1] === 'create'))).toBe(true); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1', 'https://github.com/fixture/app/pull/2']); + }); + + it('opens one pull request for a step that loops on one continue branch, and lists it once', async () => { + const { config, cwd, root, runId } = await fixture( + ` - id: change\n workspace: new\n workflow:\n steps:\n - id: fix\n workspace: continue\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 3", max_iterations: 3 }\n`, + ); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[`teamai/${runId}/change/p1`, 'main']]); + // The first pass looks the branch up before and after it opens the request, and each later pass finds it. + expect(gh.calls().filter((call) => call.args[0] === 'api')).toHaveLength(4); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('lists a pull request once when its host answers the create with another URL than its lookup gives', async () => { + const { config, cwd, runId } = await fixture( + ` - id: change\n workspace: new\n workflow:\n steps:\n - id: fix\n workspace: continue\n command: ${bump('fix.txt')}\n deliver: pr\n loop: { back_to: fix, while: "length(steps.fix.passes) < 2", max_iterations: 2 }\n`, + ); + const branch = `teamai/${runId}/change/p1`; + const open: string[] = []; + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + open.push('https://github.com/fixture/app/pull/1'); + return 'https://github.com/fixture/app/-/merge_requests/1'; + }); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(create).toHaveBeenCalledTimes(1); + expect(buildEndRecord(await state(config, runId), remote, 'tester').prs).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it.each([ + ['lists a request on another host', async (): Promise => 'https://other.example/review/1', 'https://github.com/fixture/app/pull/1', true], + ['cannot answer, and the create named a request on another host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://other.example/review/1', false], + ['cannot answer, and the create named a request on the origin host', (): Promise => Promise.reject(new Error('GitHub timed out')), 'https://github.com/fixture/app/pull/1', false], + ])('records no receipt, when the lookup after it %s', async (_case, lookupAfter, answered, unsafe) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const create = providerDouble(async () => (create.mock.calls.length === 0 ? undefined : lookupAfter()), async () => answered); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(unsafe ? `answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}.` : `may have opened a pull request from branch ${branch}, but TeamAI could not confirm it`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('starts the next for_each item while an earlier one delivers, since max_parallel bounds agents', async () => { + const { config, root, runId } = await fixture(` - id: build\n for_each: { items: inputs.packages, max_parallel: 1 }\n workspace: new\n command: node build.js\n deliver: pr\n`, + { packages: ['one', 'two'].map((name) => ({ name, id: name, after: [] })) }); + const first = await take(config, runId); + await succeed(config, runId, first, { to: pushWork(root, first), pr_url: 'https://review.acme.internal/changes/1' }); + const held = deliveryLockPath(config, runId); + expect(await acquireLock(held)).toBe(true); + try { + expect(await take(config, runId)).toMatchObject({ step: 'build[1]' }); + } finally { + await releaseLock(held); + } + }); + + it('lists the pull requests of three for_each items in item order', async () => { + const { config, cwd, root, runId } = await fixture(` - id: build\n for_each: inputs.packages\n workspace: new\n command: ${bump('items.txt')}\n deliver: pr\n`, + { packages: ['one', 'two', 'three'].map((name) => ({ name, id: name, after: [] })) }); + const gh = fakeGh(path.join(root, 'gh')); + + expect(await runLocalDriver(config, runId, cwd)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([0, 1, 2].map((index) => [`teamai/${runId}/build-${index}/p1`, 'main'])); + const run = await state(config, runId); + expect(run.deliveries?.map((receipt) => receipt.step)).toEqual(['build[0]', 'build[1]', 'build[2]']); + expect(buildEndRecord(run, remote, 'tester').prs).toEqual([1, 2, 3].map((number) => `https://github.com/fixture/app/pull/${number}`)); + }); +}); + +describe('protected paths', () => { + it('protect every path a step reads from the run base, every CI and hook path, and AGENTS.md or CLAUDE.md at any depth, in any case', () => { + const changed = [...AGENT_PATHS, ...DELIVERY_ONLY_PATHS].flatMap((root) => [root, `${root}/nested/file`, root.toUpperCase()]); + expect(protectedPaths(changed)).toEqual(changed); + expect(protectedPaths(['src/app.ts', 'docs/agents.mdx', '.claudeignore', '.github/dependabot.yml', '.github/workflows-notes.md', 'packages/web/AGENTS.md', 'Claude.md'])) + .toEqual(['packages/web/AGENTS.md', 'Claude.md']); + }); +}); + +describe('reading the delivered branch', () => { + it('tells a commit that is not an ancestor from a git error', async () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-git-code-')); + execFileSync('git', ['init', '-q', repo]); + const missing = '0'.repeat(40); + + try { + expect((await worktree.git(repo, ['merge-base', '--is-ancestor', missing, missing], { timeoutMs: 5_000 })).code).toBe(128); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } + }); +}); + +describe('reported URLs from a driver', () => { + it('drops a URL with credentials without failing the report, and never stores, echoes or publishes it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://user:s3cret-value@github.com/fixture/app/pull/1', extra: 'kept-out' }); + expect(accepted).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, + dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }], ignored: ['changes.extra'], + } }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(run.steps[0]).not.toHaveProperty('delivery.reported'); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + expect(JSON.stringify([accepted, run, buildEndRecord(run, remote, 'tester'), gh.calls()])).not.toMatch(/s3cret-value|kept-out/); + }); + + it('drops a URL from a step that does not deliver, saying why, and still accepts the report', async () => { + const { config, root, runId } = await fixture(' - id: write\n command: node write.js\n workspace: new\n'); + const gh = fakeGh(path.join(root, 'gh')); + expect(await succeed(config, runId, await take(config, runId), { pr_url: 'https://github.com/fixture/app/pull/1' })).toEqual({ data: { + accepted: true, revision: 2, step: { id: 'write', status: 'succeeded', outcome: 'success' }, cleanup: true, + dropped: [{ field: 'changes.pr_url', reason: 'the step has no deliver: pr' }], + } }); + expect((await state(config, runId)).deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + + it.each([ + ['fails', FIX_THEN_REVIEW, { outcome: 'failed' }, true, 'the step failed'], + ['succeeds with stdout that does not match its output', `${FIX_THEN_REVIEW.split(' - id: review')[0]} output:\n ok: boolean\n`, { outcome: 'success', outputs: { ok: 'yes' } }, true, 'the step failed'], + ['is released', FIX_THEN_REVIEW, { outcome: 'released' }, true, 'the step was released'], + ['succeeds with its branch at the run base', FIX_THEN_REVIEW, { outcome: 'success' }, false, 'the branch has no changes beyond the run base'], + ])('names a kept URL as dropped, without its value, when the step %s', async (_case, steps, body, pushes, reason) => { + const { config, root, runId } = await fixture(steps); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const base = fix.workspace?.base.commit; + const to = pushes ? pushWork(root, fix) : base; + const url = 'https://review.acme.internal/changes/4821'; + const accepted = await report(config, runId, { attempt: fix.attempt, revision: fix.revision, ...body, + changes: { branch: fix.workspace?.branch, base, from: base, to, commits: pushes ? [to] : [], pr_url: url } }); + + expect(accepted).toMatchObject({ data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason }] } }); + expect(JSON.stringify(accepted)).not.toContain('changes/4821'); + expect((await state(config, runId)).deliveries).toBeUndefined(); + expect(gh.calls()).toEqual([]); + }); + + it.each([ + ['is not the open pull request of its branch', [7], (branch: string) => `is not the open pull request of branch ${branch} into main`, 7], + ['names a branch with no open pull request', [], (branch: string) => `branch ${branch} has no open pull request into main`, 1], + ])('drops a URL on the origin host that %s, and delivers the open one of the branch instead', async (_case, open, reason, delivered) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + fakeGh(path.join(root, 'gh'), { open: open.map((number) => pull(number, branch)) }); + expect(await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/9' })).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: reason(branch) }], + } }); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual([`https://github.com/fixture/app/pull/${delivered}`]); + }); + + it('keeps a URL on the origin host that is the open pull request of its branch, and opens nothing for it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const gh = fakeGh(path.join(root, 'gh'), { open: [pull(4, `teamai/${runId}/fix/p1`)] }); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/4?tab=files#top' }); + expect(accepted).toMatchObject({ data: { accepted: true, step: { id: 'fix', status: 'delivering' } } }); + expect(accepted).not.toHaveProperty('data.dropped'); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/4']); + expect(gh.calls().filter((call) => call.args[0] === 'pr')).toEqual([]); + }); + + it.each([ + ['cannot answer', (): FakeGhOptions => ({ lookup: 'fail' })], + ['answers with a URL on another host', (branch: string): FakeGhOptions => ({ open: [{ ...pull(4, branch), html_url: 'https://github.example.net/fixture/app/pull/4' }] })], + ])('drops a URL on the origin host when the lookup that checks it %s, without its value, and delivers the open one later', async (_case, options) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const gh = fakeGh(path.join(root, 'gh'), options(branch)); + const accepted = await succeed(config, runId, fix, { to: pushWork(root, fix), pr_url: 'https://github.com/fixture/app/pull/4' }); + expect(accepted).toMatchObject({ data: { + accepted: true, step: { id: 'fix', status: 'delivering' }, dropped: [{ field: 'changes.pr_url', reason: `could not be checked against the open pull requests of branch ${branch} into main` }], + } }); + expect(JSON.stringify(accepted)).not.toContain('pull/4'); + expect((await state(config, runId)).steps[0]).not.toHaveProperty('delivery.reported'); + + gh.configure({ open: [pull(4, branch)] }); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/4']); + expect(creates(gh)).toEqual([]); + }); +}); + +/** + * Puts a provider double on the fixture's GitHub origin: `findOpen` answers lookups and `create` opens requests. Returns + * the create spy, so a test can show what was asked of the host. + */ +function providerDouble(findOpen: PullRequestLookup['findOpen'], create: GitProvider['createPullRequest'], hostname = 'github.com') { + const provider = new GitHubProvider(); + const creates = vi.spyOn(provider, 'createPullRequest').mockImplementation(create); + vi.spyOn(registry, 'providerForRemote').mockReturnValue({ kind: 'provider', provider, pullRequests: { findOpen }, repo: 'fixture/app', hostname }); + return creates; +} + +describe('retrying and cancelling a delivery', () => { + it('delivers the winning child branch after an end cancels a sibling delivery', async () => { + const { config, root, runId } = await fixture(` - id: ship + for_each: inputs.packages + deliver: pr + workflow: + ends: + done: succeeded + steps: + - id: loser + command: echo loser + workspace: new + deliver: pr + - id: winner + needs: [] + command: echo winner + workspace: new + end: done +`, { packages: [{ name: 'a', id: 'a', after: [] }] }); + const gh = fakeGh(path.join(root, 'gh')); + config.workflows = { tools: {}, ...config.workflows, max_parallel: 2 }; + const loser = await take(config, runId); + const winnerNext = await leaseNext(config, runId, remote, { ...writable, running: [loser.attempt] }); + if ('error' in winnerNext) throw new Error(winnerNext.error.message); + if (!winnerNext.data.lease) throw new Error(JSON.stringify(winnerNext)); + const winner = LeaseSchema.parse(winnerNext.data.lease); + await succeed(config, runId, loser, { to: pushWork(root, loser, { 'loser.txt': 'loser' }) }); + await report(config, runId, { attempt: winner.attempt, revision: winner.revision, outcome: 'success', + changes: { branch: winner.workspace?.branch, base: winner.workspace?.base.commit, from: winner.workspace?.base.commit, + to: pushWork(root, winner, { 'winner.txt': 'winner' }), commits: [] } }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'ship[0].loser')).toMatchObject({ status: 'cancelled', settledAt: 2 }); + expect(creates(gh)).toEqual([[winner.workspace?.branch, 'main']]); + expect(run.steps.find((step) => step.id === 'ship')).toMatchObject({ results: [{ branch: winner.workspace?.branch }] }); + }); + + it('starts explicit cancellation recovery from the latest recorded write on a shared branch', async () => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue + - id: recover + needs: [impl, docs] + if: steps.impl.outcome == 'cancelled' + command: echo recover + workspace: new +`); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const to = pushOnto(root, docs.workspace?.branch ?? '', { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to }); + const settledAt = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + await cancelRun(config, runId, await knownSecrets(config), 'work.impl'); + + const recovery = await take(config, runId); + expect(recovery).toMatchObject({ step: 'work.recover', workspace: { start: { commit: to } } }); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ settledAt: settledAt && 'settledAt' in settledAt ? settledAt.settledAt : undefined }); + }); + + it.each([false, true])('retries delivery at a later recorded shared-branch tip and checks its protected paths (%s)', async (protectedChange) => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const docs = await take(config, runId); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed' } }); + const file = protectedChange ? '.github/workflows/docs.yml' : 'docs.txt'; + const to = pushOnto(root, docs.workspace?.branch ?? '', { [file]: 'docs' }); + await succeed(config, runId, docs, { to }); + expect((await state(config, runId)).copies?.[docs.workspace?.branch ?? '']?.commit).toBe(to); + gh.configure({}); + await retryStep(config, runId, await knownSecrets(config), 'work.impl'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: protectedChange ? 'failed' : 'succeeded' } } }); + const run = await state(config, runId); + if (protectedChange) { + const message = `commit ${to}, which a later report recorded on the shared branch ${docs.workspace?.branch}, changed protected paths against the run base: ${file}. Step "work.impl" itself left them alone, so a retry does not run it again.`; + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(message) } }); + expect(creates(gh)).toEqual([]); + expect(run.deliveries).toBeUndefined(); + await retryStep(config, runId, await knownSecrets(config), 'work.impl'); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering', attempt: impl.attempt }); + } else { + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + expect(creates(gh)).toEqual([[docs.workspace?.branch, 'main']]); + expect(run.deliveries).toHaveLength(1); + } + }); + + it.each(['puts origin back at the step\'s own commit', 'retries the workflow step'] as const)('names remedies for a protected change a later report recorded on the shared branch, and the run accepts the one that %s', async (remedy) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const docs = await take(config, runId); + const branch = docs.workspace?.branch ?? ''; + await succeed(config, runId, docs, { to: pushOnto(root, branch, { '.github/workflows/docs.yml': 'docs' }) }); + gh.configure({}); + await retryStep(config, runId, await knownSecrets(config), 'work.impl'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ failure: { code: 'delivery_failed', message: expect.stringContaining(`put branch ${branch} on origin back at ${own}, the commit the step reported, then run: teamai workflow retry ${runId} work.impl`) } }); + expect(failure).toMatchObject({ failure: { message: expect.stringContaining(`run its steps again in a new branch: teamai workflow retry ${runId} work`) } }); + + if (remedy === 'retries the workflow step') { + expect(await retryStep(config, runId, await knownSecrets(config), 'work')).toMatchObject({ data: { retried: { step: 'work', pass: 2 } } }); + expect(await take(config, runId)).toMatchObject({ step: 'work.impl', workspace: { branch: `teamai/${runId}/work/p2` } }); + return; + } + git(root, ['--git-dir=remote.git', 'update-ref', `refs/heads/${branch}`, own]); + await retryStep(config, runId, await knownSecrets(config), 'work.impl'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + + it('blames the delivering step for its own protected change when a later report recorded more work on the shared branch', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl, { '.github/workflows/impl.yml': 'impl' }) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + await succeed(config, runId, docs, { to: pushOnto(root, docs.workspace?.branch ?? '', { '.github/workflows/docs.yml': 'docs' }) }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'protected_path', message: expect.stringContaining('Step "work.impl" changed protected paths against the run base: .github/workflows/impl.yml.') } }); + expect(creates(gh)).toEqual([]); + }); + + it('refuses a recorded later tip whose history no longer holds the delivering step\'s commit', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const clone = fs.mkdtempSync(path.join(root, 'rewrite-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + git(clone, ['reset', '-q', '--hard', `${own}~1`]); + fs.writeFileSync(path.join(clone, 'docs.txt'), 'docs'); + git(clone, ['add', '-A']); + git(clone, ['-c', 'user.name=Driver', '-c', 'user.email=driver@example.com', 'commit', '-qm', 'rewrite']); + git(clone, ['push', '-q', '--force', 'origin', branch]); + const rewritten = git(clone, ['rev-parse', 'HEAD']); + await succeed(config, runId, docs, { to: rewritten }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${rewritten}, the last one a report recorded on branch ${branch}, does not contain commit ${own} the step reported`) } }); + expect(creates(gh)).toEqual([]); + expect((await state(config, runId)).deliveries).toBeUndefined(); + }); + + it('checks only the selected recorded tip when later work removed an earlier protected change', async () => { + const { config, root, runId } = await fixture(` - id: work + workspace: new + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: docs + needs: [] + command: echo docs + workspace: continue +`); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl, { '.github/workflows/temporary.yml': 'temporary' }) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const clone = fs.mkdtempSync(path.join(root, 'correction-')); + git(root, ['clone', '-q', '-b', branch, path.join(root, 'remote.git'), clone]); + git(clone, ['rm', '.github/workflows/temporary.yml']); + const to = commitAndPush(clone, branch, { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).deliveries).toHaveLength(1); + }); + + it.each(['moved', 'missing'] as const)('names the commit the run recorded last when the shared branch on origin is %s at a retry', async (change) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const impl = await take(config, runId); + const own = pushWork(root, impl); + await succeed(config, runId, impl, { to: own }); + const docs = await take(config, runId); + const branch = docs.workspace?.branch ?? ''; + const recorded = pushOnto(root, branch, { 'docs.txt': 'docs' }); + await succeed(config, runId, docs, { to: recorded }); + if (change === 'moved') pushOnto(root, branch, { 'later.txt': 'later' }); + else git(root, ['--git-dir=remote.git', 'update-ref', '-d', `refs/heads/${branch}`]); + gh.configure({}); + await retryStep(config, runId, await knownSecrets(config), 'work.impl'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failure = (await state(config, runId)).steps.find((step) => step.id === 'work.impl'); + const remedy = change === 'moved' ? `Put branch ${branch} on origin back at ${recorded}, then run` : `push ${recorded} to it, then run`; + expect(failure).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(remedy) } }); + expect(failure).not.toMatchObject({ failure: { message: expect.stringContaining(own) } }); + expect(creates(gh)).toEqual([]); + }); + + it('keeps a step delivering while origin\'s branch is ahead of what the run recorded, and delivers it once a report records that commit', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let pushed = false; + let docs: Lease | undefined; + let ahead = ''; + const read = worktree.git; + // The sibling is leased and pushes once delivery picked work.impl, before delivery reads the branch. + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + if (!pushed && args[1].includes('ls-remote')) { + pushed = true; + docs = await take(config, runId); + ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + } + return read(...args); + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(creates(gh)).toEqual([]); + if (!docs) throw new Error('work.docs was never leased'); + await succeed(config, runId, docs, { to: ahead }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + + it('keeps waiting for a report of a commit ahead on origin while a step that writes the branch still runs, however many calls come', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + + for (let drain = 1; drain <= 3; drain += 1) { + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + } + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(creates(gh)).toEqual([]); + await succeed(config, runId, docs, { to: ahead }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + expect((await state(config, runId)).steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'succeeded' }); + }); + + it('keeps a step delivering when a report records origin\'s ahead commit while delivery reads the branch, and delivers it on the next call', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let reported = false; + const read = worktree.git; + // The sibling is leased and pushes once delivery picked work.impl, and reports right after delivery read its push. + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + if (reported || !args[1].includes('ls-remote')) return read(...args); + reported = true; + const docs = await take(config, runId); + const ahead = pushOnto(root, branch, { 'docs.txt': 'docs' }); + const answer = await read(...args); + await succeed(config, runId, docs, { to: ahead }); + return answer; + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + expect(reported).toBe(true); + expect(creates(gh)).toEqual([]); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + }); + + it('records only the canonical receipt after an unconfirmed TGit-style create and a retry', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const canonical = 'https://git.woa.com/fixture/app/merge_requests/7'; + let recovered = false; + const lookup = vi.fn(async () => { + if (recovered) return canonical; + if (create.mock.calls.length > 0) throw new Error('TGit lookup timed out'); + return undefined; + }); + const create = providerDouble(lookup, async () => 'https://git.woa.com/fixture/app/-/merge_requests/7', 'git.woa.com'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`may have opened a pull request from branch ${fix.workspace?.branch}, but TeamAI could not confirm it`) } }); + expect(failed.steps[0]).toMatchObject({ failure: { message: expect.stringContaining(`teamai workflow retry ${runId} fix`) } }); + expect(failed.deliveries).toBeUndefined(); + expect(failed.creating).toHaveLength(1); + expect(lookup).toHaveBeenCalledTimes(2); + + recovered = true; + await retryStep(config, runId, await knownSecrets(config), 'fix'); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([canonical]); + expect(buildEndRecord(run, remote, 'tester').prs).toEqual([canonical]); + expect(create).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(3); + }); + + it('records only the canonical receipt when the lookup right after a TGit-style create does not list it yet', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const canonical = 'https://git.woa.com/fixture/app/merge_requests/7'; + let listed = false; + const lookup = vi.fn(async () => (listed ? canonical : undefined)); + const create = providerDouble(lookup, async () => 'https://git.woa.com/fixture/app/-/merge_requests/7', 'git.woa.com'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const failed = await state(config, runId); + expect(failed.steps[0]).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`may have opened a pull request from branch ${fix.workspace?.branch}, but TeamAI could not confirm it`) } }); + expect(failed.deliveries).toBeUndefined(); + expect(failed.creating).toHaveLength(1); + + listed = true; + await leaseNext(config, runId, remote, writable); + await retryStep(config, runId, await knownSecrets(config), 'fix'); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([canonical]); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('retries a failed delivery in the same pass and attempt, without a lease, and opens the pull request on the next call', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh'), { lookup: 'fail' }); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + + gh.configure({}); + expect(await retryStep(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: {} }); + const retried = (await state(config, runId)).steps[0]; + expect(retried).toMatchObject({ id: 'fix', status: 'delivering', attempt: fix.attempt }); + expect(retried?.pass ?? 1).toBe(1); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(creates(gh)).toEqual([[`teamai/${runId}/fix/p1`, 'main']]); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'succeeded', attempt: fix.attempt }); + }); + + it('retries a branch that changed protected paths by running the step again, in a new pass', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix, { '.github/workflows/ci.yml': 'on: push\n' }) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + + expect(await retryStep(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: {} }); + const again = await take(config, runId); + expect(again).toMatchObject({ step: 'fix', pass: 2, workspace: { branch: `teamai/${runId}/fix/p2` } }); + expect(again.attempt).not.toBe(fix.attempt); + expect(gh.calls()).toEqual([]); + }); + + it('cancels a delivering step and keeps its branch on origin', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const gh = fakeGh(path.join(root, 'gh')); + const fix = await take(config, runId); + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + + expect(await cancelRun(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: { cancelled: { steps: ['fix'] } } }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(git(root, ['--git-dir=remote.git', 'rev-parse', `refs/heads/teamai/${runId}/fix/p1`])).toBe(to); + expect(gh.calls()).toEqual([]); + }); + + it('leaves a step delivering, opening nothing, while the run lock is held when its pull request is about to be opened', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const runLock = runLockPath(config, runId); + let holding = false; + let released = false; + const update = store.updateRun; + vi.spyOn(store, 'updateRun').mockImplementation(async (...args) => { + const result = await update(...args); + // The process that holds the run lock lets it go once an update gave up waiting for it. + if (result.status === 'busy' && holding) { + await releaseLock(runLock); + holding = false; + released = true; + } + return result; + }); + let opened: string | undefined; + const create = providerDouble(async () => { + if (!released && !holding) holding = await acquireLock(runLock); + return opened; + }, async () => { + opened = 'https://github.com/fixture/app/pull/1'; + return opened; + }); + + expect(await leaseNext(config, runId, remote, writable)).toEqual({ data: { wait: { reason: 'fix is delivering', delivering: ['fix'] }, cancelled: [] } }); + expect(released).toBe(true); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'delivering' }); + expect(create).not.toHaveBeenCalled(); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(create).toHaveBeenCalledTimes(1); + expect((await state(config, runId)).deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/fixture/app/pull/1']); + }); + + it('opens nothing when the run is cancelled while its delivery looks for an open pull request', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const create = providerDouble(async () => { + await cancelRun(config, runId, await knownSecrets(config)); + return undefined; + }, async () => 'https://github.com/fixture/app/pull/1'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it.each([ + ['pushes and records a protected commit while the lookup runs', 'lookup', true, false], + ['pushes a protected commit while the lookup runs, and records it later', 'lookup', false, false], + ['pushes and records a protected commit once delivery read origin\'s branch for the last time', 'last read', true, false], + ['pushes and records a protected commit while the lookup finds the branch\'s open pull request', 'lookup', true, true], + ] as const)('opens nothing when a sibling %s, and checks that commit on the next call', async (_case, when, recorded, open) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; + let sibling: string | undefined; + // Leased only once delivery picked work.impl: a sibling leased before the pick holds the delivery back. + const siblingPushes = async () => { + docs = await take(config, runId); + sibling = pushOnto(root, branch, { '.github/workflows/ci.yml': 'on: push\n' }); + if (recorded) await succeed(config, runId, docs, { to: sibling }); + }; + let looked = false; + const read = worktree.git; + vi.spyOn(worktree, 'git').mockImplementation(async (...args) => { + const answer = await read(...args); + if (when === 'last read' && looked && sibling === undefined && args[1].includes('ls-remote')) await siblingPushes(); + return answer; + }); + let opened: string | undefined = open ? 'https://github.com/fixture/app/pull/1' : undefined; + const create = providerDouble(async () => { + if (when === 'lookup' && sibling === undefined) await siblingPushes(); + looked = true; + return opened; + }, async () => { + opened = 'https://github.com/fixture/app/pull/1'; + return opened; + }); + + const answer = await leaseNext(config, runId, remote, writable); + if (!docs) throw new Error('work.docs was never leased'); + // A sibling that has not reported is still leased, and this call did not list it as running, so it gets it back. + expect(answer).toMatchObject({ data: recorded ? { wait: { delivering: ['work.impl'] } } : { lease: { step: 'work.docs', attempt: docs.attempt } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + + if (!recorded) await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + const message = `commit ${sibling}, which a later report recorded on the shared branch ${branch}, changed protected paths against the run base: .github/workflows/ci.yml.`; + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(message) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it('opens nothing when a sibling that writes the shared branch is leased while the lookup runs, and checks its commit once it reports', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; + const create = providerDouble(async () => { + docs ??= await take(config, runId); + return undefined; + }, async () => 'https://github.com/fixture/app/pull/1'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + + if (!docs) throw new Error('work.docs was never leased'); + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it('takes no open pull request when a sibling that writes the shared branch is leased while the lookup finds it, and checks its commit once it reports', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const branch = impl.workspace?.branch ?? ''; + let docs: Lease | undefined; + const create = providerDouble(async () => { + docs ??= await take(config, runId); + return 'https://github.com/fixture/app/pull/1'; + }, async () => 'https://github.com/fixture/app/pull/2'); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { lease: { step: 'work.docs' } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.deliveries).toBeUndefined(); + + if (!docs) throw new Error('work.docs was never leased'); + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(create).not.toHaveBeenCalled(); + }); + + it('asks the host nothing while a sibling that writes the shared branch is leased, however many calls come', async () => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const gh = fakeGh(path.join(root, 'gh')); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + const branch = docs.workspace?.branch ?? ''; + + for (let call = 1; call <= 3; call += 1) { + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + } + expect(gh.calls()).toEqual([]); + + await succeed(config, runId, docs, { to: pushOnto(root, branch, { 'docs.txt': 'docs' }) }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[branch, 'main']]); + }); + + it.each([ + ['TeamAI would open its pull request', {}, false], + ['its driver reported its pull request', { pr_url: 'https://github.com/fixture/app/pull/1' }, true], + ['the lookup finds its open pull request', {}, true], + ])('delivers nothing while a sibling that writes the shared branch is leased, when %s, and checks the sibling\'s commit once it reports', async (_case, changes, open) => { + const { config, root, runId } = await fixture(IMPL_AND_DOCS); + const impl = await take(config, runId); + const branch = impl.workspace?.branch ?? ''; + const gh = fakeGh(path.join(root, 'gh'), { open: open ? [pull(1, branch)] : [] }); + expect(await succeed(config, runId, impl, { to: pushWork(root, impl), ...changes })).not.toHaveProperty('data.dropped'); + const lock = deliveryLockPath(config, runId); + expect(await acquireLock(lock)).toBe(true); + const docs = await take(config, runId); + await releaseLock(lock); + + expect(await leaseNext(config, runId, remote, { ...writable, running: [docs.attempt] })).toMatchObject({ data: { wait: { delivering: ['work.impl'] } } }); + const waiting = await state(config, runId); + expect(waiting.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'delivering' }); + expect(waiting.creating).toBeUndefined(); + expect(waiting.deliveries).toBeUndefined(); + + const sibling = pushOnto(root, branch, { '.github/workflows/x.yml': 'on: push\n' }); + await succeed(config, runId, docs, { to: sibling }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + const run = await state(config, runId); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'failed', failure: { code: 'delivery_failed', message: expect.stringContaining(`commit ${sibling}, which a later report recorded on the shared branch ${branch}`) } }); + expect(run.deliveries).toBeUndefined(); + expect(creates(gh)).toEqual([]); + }); + + it('records a pull request opened while the run was cancelled, keeps the step cancelled, and queues the end record again with it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + let opened: string | undefined; + providerDouble(async () => opened, async () => { + await cancelRun(config, runId, await knownSecrets(config)); + opened = url; + return url; + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch: `teamai/${runId}/fix/p1`, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + }); + + it.each<[string, number]>([ + ['none', 2], ['failed', 3], ['unsafe', 1], + ])('bounds unresolved-create lookups answering %s across drains', async (answer, limit) => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let recovering = false; + const lookup = vi.fn(async () => { + if (!recovering) return undefined; + if (answer === 'failed') throw new Error('host unavailable'); + return answer === 'unsafe' ? 'https://other.example/pull/7' : undefined; + }); + const create = providerDouble(lookup, async () => { + await cancelRun(config, runId, await knownSecrets(config)); + throw new Error('create timed out'); + }); + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(2); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 0, none: 0, failed: 0 } })]); + recovering = true; + lookup.mockClear(); + + for (let count = 1; count <= limit; count += 1) { + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(count); + const run = await state(config, runId); + if (count < limit) { + expect(run.creating).toEqual([expect.objectContaining({ lookups: { total: count, none: answer === 'none' ? count : 0, failed: answer === 'failed' ? count : 0 } })]); + const projected = projectRun(run, {}); + expect(projected.ok && projected.value.creating).toEqual(run.creating); + } else expect(run.creating).toBeUndefined(); + expect(run.deliveries).toBeUndefined(); + } + await leaseNext(config, runId, remote, writable); + expect(lookup).toHaveBeenCalledTimes(limit); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('keeps an empty unresolved lookup until a later drain finds its receipt', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let url: string | undefined; + const lookup = vi.fn(async () => url); + providerDouble(lookup, async () => { + await cancelRun(config, runId, await knownSecrets(config)); + throw new Error('create timed out'); + }); + await leaseNext(config, runId, remote, writable); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + url = 'https://github.com/fixture/app/pull/7'; + await leaseNext(config, runId, remote, writable); + const run = await state(config, runId); + expect(run.creating).toBeUndefined(); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(lookup).toHaveBeenCalledTimes(4); + }); + + it('counts the lookups of a retried create from zero, so one empty lookup after it keeps the request listed', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const lookup = vi.fn(async () => undefined); + const create = providerDouble(lookup, async () => { + throw new Error('create timed out'); + }); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + + await retryStep(config, runId, await knownSecrets(config), 'fix'); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 0, none: 0, failed: 0 } })]); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + expect(create).toHaveBeenCalledTimes(2); + }); + + it('records a pull request a cancelled delivery opened before its process stopped, by looking it up on a later call, and opens no other', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + config.usageReportEnabled = true; + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + const open: string[] = []; + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + await cancelRun(config, runId, await knownSecrets(config)); + open.push(url); + opened(); + // The process stops here: the host opened the request, and the run never hears of it. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + // A stopped process leaves its delivery lock to whoever comes next. + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ step: 'fix', pass: 1, branch, target: 'main' })]); + expect((await state(config, runId)).deliveries).toBeUndefined(); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled' }); + expect(run.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect((await pendingRecords(config)).find((record) => record.run === runId && record.phase === 'end')?.prs).toEqual([url]); + expect(create).toHaveBeenCalledTimes(1); + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + expect((await state(config, runId)).deliveries).toHaveLength(1); + }); + + it('only looks up the request a cancelled delivery asked for before its process stopped, and opens none when the host holds none', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async () => undefined, async () => { + if (create.mock.calls.length > 1) return 'https://github.com/fixture/app/pull/9'; + await cancelRun(config, runId, await knownSecrets(config)); + opened(); + // The process stops here, before the host opened anything. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect((await state(config, runId)).steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect((await state(config, runId)).creating).toEqual([expect.objectContaining({ step: 'fix', pass: 1, branch, target: 'main' })]); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: {} } }); + const run = await state(config, runId); + expect(create).toHaveBeenCalledTimes(1); + expect(run.deliveries).toBeUndefined(); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'cancelled', delivery: { branch } }); + expect(run.creating).toEqual([expect.objectContaining({ lookups: { total: 1, none: 1, failed: 0 } })]); + await leaseNext(config, runId, remote, writable); + expect((await state(config, runId)).creating).toBeUndefined(); + }); + + it('records a pull request a cancelled delivery opened before its process stopped, when the step is retried before any later call', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/8'; + const open: string[] = []; + let opened = (): void => {}; + const creating = new Promise((resolve) => { opened = resolve; }); + const create = providerDouble(async ({ source, target }) => (source === branch && target === 'main' ? open[0] : undefined), async () => { + await cancelRun(config, runId, await knownSecrets(config), 'fix'); + open.push(url); + opened(); + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + expect(await retryStep(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: { retried: { step: 'fix', pass: 2 } } }); + + expect(await take(config, runId)).toMatchObject({ step: 'fix', pass: 2 }); + expect((await state(config, runId)).deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect(create).toHaveBeenCalledTimes(1); + }); + + it('leaves a step that was retried while a lookup-only reconcile ran delivering, and delivers it', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + await succeed(config, runId, fix, { to: pushWork(root, fix) }); + const url = 'https://github.com/fixture/app/pull/3'; + let retried = false; + let opened: string | undefined; + const create = providerDouble(async () => { + const step = (await state(config, runId)).steps[0]; + if (step?.status === 'failed' && !retried) { + retried = true; + expect(await retryStep(config, runId, await knownSecrets(config), 'fix')).toMatchObject({ data: { retried: { step: 'fix', pass: 1 } } }); + return undefined; + } + if (create.mock.calls.length === 1 && !retried) throw new Error('GitHub timed out'); + return opened; + }, async () => { + if (create.mock.calls.length === 1) throw new Error('GitHub timed out'); + opened = url; + return url; + }); + + expect(await leaseNext(config, runId, remote, writable)).toMatchObject({ data: { done: { outcome: 'failed' } } }); + expect(retried).toBe(false); + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + const run = await state(config, runId); + expect(retried).toBe(true); + expect(run.steps[0]).toMatchObject({ id: 'fix', status: 'succeeded' }); + expect(run.deliveries?.map((receipt) => receipt.url)).toEqual([url]); + expect(create).toHaveBeenCalledTimes(2); + }); + + it('looks up before it opens again a pull request of a step that still delivers after its process stopped inside the create', async () => { + const { config, root, runId } = await fixture(FIX_THEN_REVIEW); + const fix = await take(config, runId); + const branch = `teamai/${runId}/fix/p1`; + const to = pushWork(root, fix); + await succeed(config, runId, fix, { to }); + const url = 'https://github.com/fixture/app/pull/4'; + const asked: string[] = []; + const open: string[] = []; + let stopped = (): void => {}; + const creating = new Promise((resolve) => { stopped = resolve; }); + const create = providerDouble(async () => { + asked.push('find'); + return open[0]; + }, async () => { + asked.push('create'); + if (create.mock.calls.length > 1) { + open.push(url); + return url; + } + stopped(); + // The process stops here, before the host opened anything, and the step still delivers. + return new Promise(() => {}); + }); + void leaseNext(config, runId, remote, writable); + await creating; + fs.rmSync(deliveryLockPath(config, runId), { force: true }); + const marked = await state(config, runId); + expect(marked.steps[0]).toMatchObject({ id: 'fix', status: 'delivering' }); + expect(marked.creating).toEqual([{ step: 'fix', pass: 1, settledAt: 2, branch, head: to, target: 'main', lookups: { total: 0, none: 0, failed: 0 } }]); + const handed = projectRun(marked, {}); + expect(handed.ok && handed.value.creating).toEqual(marked.creating); + + expect(await take(config, runId)).toMatchObject({ step: 'review' }); + expect(asked).toEqual(['find', 'create', 'find', 'create', 'find']); + const run = await state(config, runId); + expect(run.deliveries).toEqual([{ url, branch, base: 'main', step: 'fix', pass: 1 }]); + expect(run.creating).toBeUndefined(); + }); +}); + +describe('limits that stop a delivering run', () => { + const at = (minutes: number): Date => new Date(Date.UTC(2026, 0, 1) + minutes * 60_000); + const TIMED_WORK = ` - id: work + workspace: new + timeout: 10m + workflow: + steps: + - id: impl + command: echo impl + workspace: continue + deliver: pr + - id: after + needs: [work] + command: echo after +`; + + it.each([ + ['while its step delivers', 'before'], + ['while its delivery looks for an open pull request', 'lookup'], + ] as const)('opens nothing once a timeout passes %s', async (_case, when) => { + vi.useFakeTimers({ toFake: ['Date'] }); + try { + vi.setSystemTime(at(0)); + const { config, root, runId } = await fixture(TIMED_WORK); + const impl = await take(config, runId); + await succeed(config, runId, impl, { to: pushWork(root, impl) }); + const lookup = vi.fn(async () => { + if (when === 'lookup') vi.setSystemTime(at(15)); + return undefined; + }); + const create = providerDouble(lookup, async () => 'https://github.com/fixture/app/pull/1'); + if (when === 'before') vi.setSystemTime(at(15)); + + await leaseNext(config, runId, remote, writable); + const run = await state(config, runId); + expect(create).not.toHaveBeenCalled(); + expect(run.deliveries).toBeUndefined(); + expect(run.creating).toBeUndefined(); + expect(run.steps.find((step) => step.id === 'work')).toMatchObject({ status: 'failed', failure: { code: 'timeout' } }); + expect(run.steps.find((step) => step.id === 'work.impl')).toMatchObject({ status: 'cancelled' }); + if (when === 'before') expect(lookup).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/src/__tests__/workflow-docs-examples.test.ts b/src/__tests__/workflow-docs-examples.test.ts index 53d02627c..9d4f3f37b 100644 --- a/src/__tests__/workflow-docs-examples.test.ts +++ b/src/__tests__/workflow-docs-examples.test.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; import { parseDocument } from 'yaml'; import { syncScorerDefinitions } from '../scorers/cache.js'; +import { ciRecipesOf, runCommandsOf } from './helpers/ci-recipe.js'; import { formatScorerDiagnostic } from '../scorers/definition.js'; import type { LocalConfig } from '../types.js'; import { formatUnknownScorer, syncWorkflowDefinitions } from '../workflow/cache.js'; @@ -76,6 +77,8 @@ function startsOf(page: string): { starts: Start[]; unread: string[] } { const unread = [...text.matchAll(/--inputs[ =]['"{]/g)] .map((match) => lineOf(match.index)) .filter((line) => !starts.some((start) => start.line === line)) + // Actions expressions become JSON only inside the job; the documented CLI path has its own real-CLI E2E. + .filter((line) => !text.split('\n')[line - 1]?.includes('${{')) .map((line) => `${page}:${line}: --inputs is not part of a teamai workflow start line`); return { starts, unread }; } @@ -308,6 +311,19 @@ describe('workflow guide examples', () => { expect(JSON.parse(prompt.slice(prompt.indexOf('[')))).toEqual(shown); }); + it.each(PAGES)('%s: every job of a GitHub Actions workflow it shows says what it runs on', (page) => { + const jobs = ciRecipesOf(path.join(ROOT, page)).flatMap((recipe) => Object.entries(recipe.jobs)); + expect(jobs.filter(([, job]) => !job['runs-on']).map(([name]) => `${page}: job ${name} has no runs-on`)).toEqual([]); + }); + + it('shows the same CI recipe commands in every language', () => { + const recipes = PAGES.filter((page) => path.posix.basename(page) === 'workflow.md') + .map((page) => ({ page, commands: ciRecipesOf(path.join(ROOT, page)).map(runCommandsOf) })); + expect(recipes.map(({ page }) => page)).toEqual(['docs/guide/workflow.md', 'docs/guide/zh-CN/workflow.md']); + expect(recipes[0]?.commands.flat().length, 'the English guide shows a CI recipe with run commands').toBeGreaterThan(0); + for (const { page, commands } of recipes.slice(1)) expect(commands, page).toEqual(recipes[0]?.commands); + }); + it.each([...PAGES, ...SCORER_PAGES])('%s: every relative link to a .md file resolves', (page) => { const text = fs.readFileSync(path.join(ROOT, page), 'utf8'); const missing = [...text.matchAll(/\]\(([^)#\s:]+\.md)(?:#[^)]*)?\)/g)] diff --git a/src/__tests__/workflow-engine.test.ts b/src/__tests__/workflow-engine.test.ts index cd6f85726..6917c6eba 100644 --- a/src/__tests__/workflow-engine.test.ts +++ b/src/__tests__/workflow-engine.test.ts @@ -4737,6 +4737,124 @@ describe('the session a step resumes after other steps continued it', () => { }); }); +describe('deliver: pr at workflow start', () => { + async function start(files: Record, name: string, base?: RunState['base']): Promise<{ config: LocalConfig; started: Awaited> }> { + const { config } = testConfig(); + for (const [file, body] of Object.entries(files)) fs.writeFileSync(path.join(config.repo.localPath, 'workflows', `${file}.yaml`), body); + await syncWorkflowDefinitions(config); + return { config, started: await startRun(config, name, {}, { secrets: await knownSecrets(config), ...(base ? { base } : {}) }) }; + } + const commit = 'a'.repeat(40); + + it('refuses a top-level prompt that delivers but runs read-only, naming the step, before the run exists', async () => { + const { config, started } = await start({ ship: workflow('ship', ` - id: fix + prompt: Fix the bug. + deliver: pr +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toEqual({ error: { + code: 'bad_delivery', + message: 'Step "fix" has deliver: pr but runs read-only. Set workspace: new on it, or on the workflow step that runs it.', + } }); + expect(fs.existsSync(path.join(config.dataHome ?? '', 'workflows', 'runs'))).toBe(false); + }); + + it('starts the same prompt inside a sub-workflow that a workspace: new step runs', async () => { + const { started } = await start({ + fix: workflow('fix', ` - id: write + prompt: Fix the bug. + deliver: pr +`), + ship: workflow('ship', ` - id: change + workspace: new + workflow: fix +`), + }, 'ship', { ref: 'main', commit }); + + expect(started).toMatchObject({ data: { status: 'running' } }); + }); + + it('refuses a workflow step that delivers without a copy of its own', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: change + deliver: pr + workflow: + steps: + - id: write + workspace: new + prompt: Fix the bug. +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toEqual({ error: { + code: 'bad_delivery', + message: 'Step "change" delivers its workflow, so it needs workspace: new to own the branch it delivers.', + } }); + }); + + it.each([ + ['refuses', '', { error: { + code: 'bad_delivery', + message: 'Step "each" has deliver: pr, but no step of the workflow it runs writes a branch, so it would deliver nothing. Set workspace: new on "each" so its prompt steps write, or give a step in that workflow workspace: new or continue.', + } }], + ['starts', ' workspace: new\n', { data: { status: 'running' } }], + ])('%s a for_each workflow step that delivers, by whether a step its items run writes a branch', async (_case, workspace, expected) => { + const { config } = testConfig(); + fs.writeFileSync(path.join(config.repo.localPath, 'workflows', 'review.yaml'), workflow('review', ` - id: read + prompt: Review the package. + - id: lint + needs: [read] + command: npm run lint +`)); + fs.writeFileSync(path.join(config.repo.localPath, 'workflows', 'ship.yaml'), workflow('ship', ` - id: each + for_each: inputs.packages +${workspace} deliver: pr + workflow: review +`, 'inputs: { packages: [string] }\n')); + expect((await syncWorkflowDefinitions(config)).invalid).toEqual([]); + + expect(await startRun(config, 'ship', { packages: ['web'] }, { base: { ref: 'main', commit }, secrets: await knownSecrets(config) })).toMatchObject(expected); + }); + + it.each(['approve: Continue?', 'end: done'])('refuses delivery when an inherited writable workflow contains only approval or end steps (%s)', async (action) => { + const { started } = await start({ ship: workflow('ship', ` - id: change + workspace: new + deliver: pr + workflow: + ends: { done: succeeded } + steps: + - id: approve + ${action} + - id: check + needs: [] + command: echo check + end: done +`) }, 'ship', { ref: 'main', commit }); + + expect(started).toMatchObject({ error: { code: 'bad_delivery', message: expect.stringContaining('no step of the workflow it runs writes a branch') } }); + }); + + it('refuses a run that delivers from a detached HEAD, telling to start from a named branch', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: fix + workspace: new + prompt: Fix the bug. + deliver: pr +`) }, 'ship', { commit }); + + expect(started).toEqual({ error: { + code: 'no_base_branch', + message: `Step "fix" delivers a pull request, but the run would start from a detached HEAD at ${commit}, so no branch is there for the pull request to target. Check out a named branch, then start the run again.`, + } }); + }); + + it('starts a run from a detached HEAD when no step delivers', async () => { + const { started } = await start({ ship: workflow('ship', ` - id: fix + workspace: new + prompt: Fix the bug. +`) }, 'ship', { commit }); + + expect(started).toMatchObject({ data: { status: 'running' } }); + }); +}); + describe('deadlines', () => { const at = (minutes: number): string => new Date(Date.UTC(2026, 0, 1) + minutes * 60_000).toISOString(); diff --git a/src/__tests__/workflow-handoff.test.ts b/src/__tests__/workflow-handoff.test.ts index 3635627a5..74ff2ad9b 100644 --- a/src/__tests__/workflow-handoff.test.ts +++ b/src/__tests__/workflow-handoff.test.ts @@ -23,6 +23,7 @@ import { readRun, runLockPath, updateRun } from '../workflow/store.js'; import type { DriverIdentity, RunState, WorkflowEnvelope } from '../workflow/types.js'; import { ensureWorktree, fetchMissingCommits, headCommit, markReadCopy, worktreePath } from '../workflow/worktree.js'; import { knownSecrets } from '../env-resolution.js'; +import { fakeGh } from './helpers/fake-gh.js'; const dirs: string[] = []; @@ -93,6 +94,34 @@ steps: command: echo change > change.txt `; +const STACK = `schemaVersion: v1alpha1 +name: stack +description: A change, and one that starts from it. +steps: + - id: a + workspace: new + command: echo a > a.txt + - id: b + needs: [a] + workspace: new + command: echo b > b.txt +`; + +const DELIVERED_STACK = `schemaVersion: v1alpha1 +name: stack +description: A change delivered as a pull request, and one stacked on it. +steps: + - id: a + workspace: new + command: echo a > a.txt + deliver: pr + - id: b + needs: [a] + workspace: new + command: echo b > b.txt + deliver: pr +`; + const WORKSPACES = ['read-only', 'new', 'continue'] as const; interface Member { @@ -321,6 +350,38 @@ async function runStep(member: Member, runId: string, outputs?: Record): string[][] { + return gh.calls().filter((call) => call.args[0] === 'pr').map((call) => ['-H', '-B'].map((flag) => call.args[call.args.indexOf(flag) + 1] ?? '')); +} + +/** + * Starts the delivered stack on `alice`, delivers `a` and reports `b`, so `b` is delivering when the run is published. + * The fake host keeps `a`'s pull request. + */ +async function deliveringStack(acme: ReturnType, alice: Member, bob: Member) { + onGitHub(acme.business, [alice, bob]); + const gh = fakeGh(path.join(path.dirname(acme.origin), 'gh')); + const runId = await startIn(alice, 'stack'); + const a = await work(alice, runId, 'a.txt'); + const b = await work(alice, runId, 'b.txt'); + return { gh, runId, a, b }; +} + describe('workflow handoff through teamai-reports', () => { it('hands a published run to another install, which continues it, and refuses the earlier owner', async () => { const acme = team(); @@ -519,7 +580,7 @@ describe('workflow handoff through teamai-reports', () => { }); interface HandedStep { id: string; changes: { branch: string; base: string; from: string; to: string; commits: string[] }; read?: unknown; passes?: unknown[] } - interface HandedRun { base: { commit: string }; copies: Record; steps: HandedStep[]; archive: unknown[] } + interface HandedRun { base: { commit: string }; copies: Record; steps: HandedStep[]; archive: unknown[] } it.each<[string, (snapshot: HandedRun, step: HandedStep, branch: string) => void, (at: number, branch: string) => string]>([ ['its base', (snapshot) => { snapshot.base.commit = 'main'; }, () => 'base.commit'], ['a copy', (snapshot, _step, branch) => { snapshot.copies[branch] = { commit: 'HEAD' }; }, (_at, branch) => `copies.${branch}.commit`], @@ -561,9 +622,13 @@ describe('workflow handoff through teamai-reports', () => { expect(await readRun(bob.config, runId)).toBe('missing'); }); - it.each([['a branch outside any run', 'main'], ['a branch of another run', 'teamai/r0123456789abcdef/change/p1']])( - 'refuses to claim a snapshot whose copy is %s, and changes nothing', - async (_name, branch) => { + it.each<[string, (branch: string, commit: string) => HandedRun['copies'], (branch: string) => string]>([ + ['is a branch outside any run', (_branch, commit) => ({ main: { commit } }), () => 'copies.main'], + ['is a branch of another run', (_branch, commit) => ({ 'teamai/r0123456789abcdef/change/p1': { commit } }), () => 'copies.teamai/r0123456789abcdef/change/p1'], + ['started from a branch outside the run', (branch, commit) => ({ [branch]: { commit, from: 'main' } }), (branch) => `copies.${branch}.from`], + ])( + 'refuses to claim a snapshot whose copy %s, and changes nothing', + async (_name, copies, field) => { const acme = team({ ship: SHIP }); const alice = await acme.member('alice'); const bob = await acme.member('bob'); @@ -574,12 +639,12 @@ describe('workflow handoff through teamai-reports', () => { git(path.dirname(acme.origin), 'clone', '-q', '--branch', 'teamai-reports', acme.origin, editor); const rel = `workflows/runs/${runId}/state.json`; const snapshot: HandedRun = JSON.parse(fs.readFileSync(path.join(editor, rel), 'utf8')); - snapshot.copies = { [branch]: { commit: wrote.commit } }; + snapshot.copies = copies(wrote.branch, wrote.commit); fs.writeFileSync(path.join(editor, rel), `${JSON.stringify(snapshot, null, 2)}\n`); git(editor, 'commit', '-qam', 'hand edit'); git(editor, 'push', '-q', 'origin', 'teamai-reports'); - expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field: `copies.${branch}` } }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field: field(wrote.branch) } }); expect(await readRun(bob.config, runId)).toBe('missing'); }, ); @@ -1318,6 +1383,80 @@ describe('workflow handoff through teamai-reports', () => { expect(git(bob.checkout, 'rev-parse', `${more.commit}^`)).toBe(wrote.commit); }); + it('hands over a copy that started from the branch of a step it needs, with the branch it started from', async () => { + const acme = team({ stack: STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const runId = await startIn(alice, 'stack'); + const a = await work(alice, runId, 'a.txt'); + const b = await work(alice, runId, 'b.txt'); + + expect(await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout })).toMatchObject({ ok: true, value: { publication: 'published' } }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + const claimed = await readRun(bob.config, runId); + expect(typeof claimed !== 'string' && claimed.copies?.[b.branch]).toEqual({ commit: b.commit, from: a.branch }); + }); + + it('publishes a step that delivers with what it delivers, and the claim finds the pull request the first machine opened and opens no other', async () => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { gh, runId, a, b } = await deliveringStack(acme, alice, bob); + // Alice's process opened b's request too, then stopped before the run heard of it. + gh.configure({ open: [openPull(1, a.branch), openPull(2, b.branch, a.branch)] }); + + expect(await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout })).toMatchObject({ ok: true, value: { publication: 'published' } }); + expect(JSON.parse(originFile(acme.origin, `workflows/runs/${runId}/state.json`) ?? 'null')).toMatchObject({ + copies: { [b.branch]: { commit: b.commit, from: a.branch } }, + deliveries: [{ url: 'https://github.com/acme/app/pull/1', branch: a.branch, base: 'main', step: 'a', pass: 1 }], + steps: [{ id: 'a', status: 'succeeded' }, { id: 'b', status: 'delivering', delivery: { branch: b.branch, head: b.commit } }], + }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + + expect(await leaseNext(bob.config, runId, bob.driver)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + const run = await readRun(bob.config, runId); + expect(typeof run !== 'string' && run.steps[1]).toMatchObject({ id: 'b', status: 'succeeded' }); + expect(typeof run !== 'string' && run.deliveries?.[1]).toEqual({ url: 'https://github.com/acme/app/pull/2', branch: b.branch, base: a.branch, step: 'b', pass: 1 }); + expect(creates(gh)).toEqual([[a.branch, 'main']]); + }); + + it('opens the pull request of a step that was delivering when the run was claimed, once, on the machine that claimed it', async () => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { gh, runId, a, b } = await deliveringStack(acme, alice, bob); + await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout }); + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: true, value: { ownership: 'claimed' } }); + expect(creates(gh)).toEqual([[a.branch, 'main']]); + + expect(await leaseNext(bob.config, runId, bob.driver)).toMatchObject({ data: { done: { outcome: 'succeeded' } } }); + expect(creates(gh)).toEqual([[a.branch, 'main'], [b.branch, a.branch]]); + const run = await readRun(bob.config, runId); + expect(typeof run !== 'string' && run.deliveries?.map((receipt) => receipt.url)).toEqual(['https://github.com/acme/app/pull/1', 'https://github.com/acme/app/pull/2']); + }); + + it.each<[string, (snapshot: { deliveries: Array<{ url: string }>; steps: Array<{ delivery?: { reported?: string } }> }) => void, string]>([ + ['a receipt', (snapshot) => { snapshot.deliveries = snapshot.deliveries.map((receipt) => ({ ...receipt, url: 'https://user:token@github.com/acme/app/pull/1' })); }, 'deliveries[0].url'], + ['a reported URL', (snapshot) => { const step = snapshot.steps[1]; if (step?.delivery) step.delivery.reported = 'http://github.com/acme/app/pull/2'; }, 'steps[1].delivery.reported'], + ])('refuses to claim a snapshot whose pull request URL in %s is not one TeamAI keeps, naming the field', async (_name, edit, field) => { + const acme = team({ stack: DELIVERED_STACK }); + const alice = await acme.member('alice'); + const bob = await acme.member('bob'); + const { runId } = await deliveringStack(acme, alice, bob); + await publishRun(alice.config, runId, { env: {}, checkout: alice.checkout }); + const editor = path.join(path.dirname(acme.origin), 'editor'); + git(path.dirname(acme.origin), 'clone', '-q', '--branch', 'teamai-reports', acme.origin, editor); + const rel = `workflows/runs/${runId}/state.json`; + const snapshot = JSON.parse(fs.readFileSync(path.join(editor, rel), 'utf8')); + edit(snapshot); + fs.writeFileSync(path.join(editor, rel), `${JSON.stringify(snapshot, null, 2)}\n`); + git(editor, 'commit', '-qam', 'hand edit'); + git(editor, 'push', '-q', 'origin', 'teamai-reports'); + + expect(await claimRun(bob.config, runId, { checkout: bob.checkout })).toMatchObject({ ok: false, error: { code: 'snapshot_unsupported', field } }); + expect(await readRun(bob.config, runId)).toBe('missing'); + }); + it('refuses to publish, telling to publish again, when the run moves while publish checks the business origin', async () => { const acme = team({ ship: SHIP }); const alice = await acme.member('alice'); diff --git a/src/__tests__/workflow-parse.test.ts b/src/__tests__/workflow-parse.test.ts index 0719ce03e..f844bf8ee 100644 --- a/src/__tests__/workflow-parse.test.ts +++ b/src/__tests__/workflow-parse.test.ts @@ -427,6 +427,36 @@ steps: loop: { back_to: review, max_iterations: 3 } `; +describe('deliver', () => { + const FIX = `schemaVersion: v1alpha1 +name: build +description: Fix a bug. +steps: + - id: fix +`; + + it('parses deliver: pr on a step that writes a branch', () => { + const result = parse(`${FIX} command: node fix.js + workspace: new + deliver: pr +`); + expect(result.ok && result.definition.steps[0]).toMatchObject({ deliver: 'pr' }); + }); + + it('rejects a delivery other than pr, and deliver: pr on a step that can only read', () => { + expect(rejected(`${FIX} prompt: Fix it. + deliver: merge +`)).toEqual([{ message: `${FILE}:7: field "steps[0].deliver": step "fix" deliver must be pr; TeamAI opens a pull request and never merges`, code: 'bad_delivery' }]); + expect(rejected(`${FIX} prompt: Fix it. + workspace: read-only + deliver: pr +`)).toEqual([{ message: `${FILE}:8: field "steps[0].deliver": step "fix" has deliver: pr but runs read-only; set workspace: new so it has a branch to deliver`, code: 'bad_delivery' }]); + expect(rejected(`${FIX} command: node fix.js + deliver: pr +`)).toEqual([{ message: `${FILE}:7: field "steps[0].deliver": step "fix" has deliver: pr but runs read-only, as a command without workspace does; set workspace: new so it has a branch to deliver`, code: 'bad_delivery' }]); + }); +}); + describe('step loops', () => { it('parses a loop back to a step the owner needs, with an optional while', () => { const result = parse(`${REVIEW_AND_FIX} - id: decide diff --git a/src/__tests__/workflow-pr-providers.test.ts b/src/__tests__/workflow-pr-providers.test.ts new file mode 100644 index 000000000..7930386f1 --- /dev/null +++ b/src/__tests__/workflow-pr-providers.test.ts @@ -0,0 +1,272 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { GitHubProvider } from '../providers/github/index.js'; +import { GitLabProvider } from '../providers/gitlab/index.js'; +import { TGitProvider } from '../providers/tgit/index.js'; +import { GitCodeProvider } from '../providers/gitcode/index.js'; +import { GenericGitProvider } from '../providers/git/index.js'; +import { providerForRemote } from '../providers/registry.js'; +import type { GitProvider, PullRequestLookup, PullRequestRef } from '../providers/types.js'; +import { fakeGh, resolveGh } from './helpers/fake-gh.js'; + +const ref: PullRequestRef = { repo: 'acme/app', source: 'teamai/run/work/p1', target: 'main', timeoutMs: 1000 }; +const pull = (head: string, repo = 'acme/app', number = 7) => ({ + state: 'open', html_url: `https://github.com/acme/app/pull/${number}`, head: { ref: head, repo: { full_name: repo } }, base: { ref: 'main', repo: { full_name: 'acme/app' } }, +}); + +const dirs: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); vi.unstubAllGlobals(); vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +function lookup(): PullRequestLookup { + const found = new GitHubProvider().pullRequests; + if (!found) throw new Error('GitHub has no pull request lookup'); + return found; +} + +describe('GitHub pull request lookup', () => { + it('finds the open request of this repository branch into the target, never a fork with the same branch name', async () => { + resolveGh(null); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify([pull(ref.source, 'fork/app', 6), pull('other'), pull(ref.source)]))); + vi.stubGlobal('fetch', fetcher); + + expect(await lookup().findOpen(ref)).toBe('https://github.com/acme/app/pull/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([url.pathname, url.searchParams.get('state'), url.searchParams.get('head'), url.searchParams.get('base')]) + .toEqual(['/repos/acme/app/pulls', 'open', `acme:${ref.source}`, 'main']); + fetcher.mockResolvedValueOnce(new Response(JSON.stringify([pull(ref.source, 'fork/app', 6)]))); + expect(await lookup().findOpen(ref)).toBeUndefined(); + }); + + it('asks an installed gh with prompts off, and never logs in', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-fake-gh-')); + dirs.push(dir); + const gh = fakeGh(dir, { open: [pull(ref.source)] }); + + expect(await lookup().findOpen(ref)).toBe('https://github.com/acme/app/pull/7'); + expect(gh.calls()).toMatchObject([{ args: ['api', expect.stringMatching(/^repos\/acme\/app\/pulls\?/)], prompt: '0' }]); + }); + + it('keeps the process running while gh answers, and gives up on a gh that does not answer in time', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-fake-gh-')); + dirs.push(dir); + fakeGh(dir, { hold: 'lookup', open: [pull(ref.source)] }); + await expect(lookup().findOpen({ ...ref, timeoutMs: 300 })).rejects.toThrow('gh could not look up open pull requests'); + + // Only a timer of this process lets gh answer, so a lookup that blocked the process would time out instead. + const release = setTimeout(() => fs.writeFileSync(path.join(dir, 'release'), ''), 50); + try { + expect(await lookup().findOpen({ ...ref, timeoutMs: 5000 })).toBe('https://github.com/acme/app/pull/7'); + } finally { + clearTimeout(release); + } + }); + + it('ends the lookup at its deadline when a launcher child keeps the output pipe open', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-gh-launcher-')); + dirs.push(dir); + const launcher = path.join(dir, 'gh'); + const pidFile = path.join(dir, 'holder.pid'); + fs.writeFileSync(launcher, `#!${process.execPath} +const { spawn } = require('node:child_process'); +const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'inherit' }); +require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(child.pid)); +setInterval(() => {}, 1000); +`, { mode: 0o755 }); + resolveGh(launcher); + let deadline: ReturnType | undefined; + const answer = lookup().findOpen({ ...ref, timeoutMs: 300 }).then(() => 'found', () => 'failed'); + try { + await vi.waitFor(() => expect(fs.existsSync(pidFile)).toBe(true)); + const result = await Promise.race([ + answer, + new Promise((resolve) => { deadline = setTimeout(() => resolve('still waiting for the pipe'), 1000); }), + ]); + expect(result).toBe('failed'); + } finally { + clearTimeout(deadline); + const pid = Number(fs.readFileSync(pidFile, 'utf8')); + process.kill(pid, 'SIGKILL'); + await answer; + } + }); + + it('fails, rather than finding nothing, without credentials or on an answer it cannot read, and never repeats what GitHub said', async () => { + resolveGh(null); + vi.stubEnv('GITHUB_TOKEN', ''); vi.stubEnv('GH_TOKEN', ''); + await expect(lookup().findOpen(ref)).rejects.toThrow('GitHub credentials are unavailable'); + vi.stubEnv('GITHUB_TOKEN', 'test-token'); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('private-token-in-body', { status: 403 }))); + await expect(lookup().findOpen(ref)).rejects.toThrow(/^GitHub answered the open pull request lookup with HTTP 403\.(?!.*private-token-in-body)/); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(JSON.stringify({ message: 'private-token-in-body' })))); + await expect(lookup().findOpen(ref)).rejects.toThrow(/^GitHub answered the open pull request lookup with data TeamAI cannot read\.$/); + }); +}); + +function lookupOf(provider: GitProvider): PullRequestLookup { + if (!provider.pullRequests) throw new Error(`${provider.name} has no pull request lookup`); + return provider.pullRequests; +} + +/** Answers every request with `body` as JSON and `status`, and returns the spy. */ +function answering(body: unknown, status = 200) { + const fetcher = vi.fn().mockImplementation(async () => new Response(JSON.stringify(body), { status })); + vi.stubGlobal('fetch', fetcher); + return fetcher; +} + +/** Answers no request until its signal aborts it, as a host that never responds. */ +function hanging(): void { + vi.stubGlobal('fetch', vi.fn((_url, init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }))); +} + +/** An open merge request as the GitLab API lists it. */ +const mr = (source: string, iid = 7, sourceProject = 1, url = `https://gitlab.com/acme/app/-/merge_requests/${iid}`) => ({ + web_url: url, state: 'opened', source_branch: source, target_branch: 'main', source_project_id: sourceProject, target_project_id: 1, +}); + +describe('GitLab merge request lookup', () => { + beforeEach(() => { + vi.stubEnv('GITLAB_TOKEN', 'test-token'); + vi.stubEnv('GITLAB_URL', ''); + vi.stubEnv('TEAMAI_GITLAB_HOST', ''); + }); + + it('finds the open request of this project branch into the target, never one from a fork with the same branch name', async () => { + const fetcher = answering([mr(ref.source, 6, 2), mr('other', 8), mr(ref.source)]); + + expect(await lookupOf(new GitLabProvider()).findOpen(ref)).toBe('https://gitlab.com/acme/app/-/merge_requests/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('source_branch'), url.searchParams.get('target_branch')]) + .toEqual(['https://gitlab.com/api/v4/projects/acme%2Fapp/merge_requests', 'opened', ref.source, 'main']); + answering([mr(ref.source, 6, 2)]); + expect(await lookupOf(new GitLabProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, without a token, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new GitLabProvider()); + vi.stubEnv('GITLAB_TOKEN', ''); vi.stubEnv('GITLAB_PRIVATE_TOKEN', ''); vi.stubEnv('GITLAB_PAT', ''); + await expect(lookup.findOpen(ref)).rejects.toThrow('GitLab authentication unavailable'); + vi.stubEnv('GITLAB_TOKEN', 'test-token'); + answering({ message: 'private-token-in-body' }, 403); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitLab answered the open merge request lookup with HTTP 403\.(?!.*private-token-in-body)/); + answering({ message: 'private-token-in-body' }); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitLab answered the open merge request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => mr(`other-${index}`, index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open merge requests than one page'); + answering([mr(ref.source, 7, 1, 'https://evil.example/acme/app/-/merge_requests/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); + + it('gives up on a create GitLab does not answer in time', async () => { + hanging(); + await expect(new GitLabProvider().createPullRequest({ repo: 'acme/app', source: ref.source, target: 'main', title: 't', spawnTimeoutMs: 20 })).rejects.toThrow(); + }); +}); + +describe('TGit merge request lookup', () => { + const tgitMr = (source: string, iid = 7, sourceProject = 1, state = 'opened', url = `https://git.woa.com/acme/app/merge_requests/${iid}`) => ({ + ...mr(source, iid, sourceProject, url), state, + }); + beforeEach(() => vi.stubEnv('TGIT_TOKEN', 'test-token')); + + it('finds the open or reopened request of this project branch into the target, never one from a fork with the same branch name', async () => { + const fetcher = answering([tgitMr(ref.source, 6, 2), tgitMr('other', 8), tgitMr(ref.source, 9, 1, 'reopened')]); + + expect(await lookupOf(new TGitProvider()).findOpen(ref)).toBe('https://git.woa.com/acme/app/merge_requests/9'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('source_branch'), url.searchParams.get('target_branch')]) + .toEqual(['https://git.woa.com/api/v3/projects/acme%2Fapp/merge_requests', 'opened', ref.source, 'main']); + answering([tgitMr(ref.source, 6, 2), tgitMr(ref.source, 5, 1, 'merged')]); + expect(await lookupOf(new TGitProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new TGitProvider()); + answering({ message: 'private-token-in-body' }, 404); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^TGit answered the open merge request lookup with HTTP 404\.(?!.*private-token-in-body)/); + answering([{ web_url: 'https://git.woa.com/acme/app/merge_requests/7', state: 'opened' }]); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^TGit answered the open merge request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => tgitMr(`other-${index}`, index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open merge requests than one page'); + answering([tgitMr(ref.source, 7, 1, 'opened', 'https://evil.example/acme/app/merge_requests/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); +}); + +describe('GitCode pull request lookup', () => { + const gitcodePull = (head: string, repo = 'acme/app', number = 7, state = 'open', url = `https://gitcode.com/acme/app/pull/${number}`) => ({ + ...pull(head, repo, number), html_url: url, state, + }); + beforeEach(() => vi.stubEnv('GITCODE_TOKEN', 'test-token')); + + it('finds the open request of this repository branch into the target, never a fork with the same branch name', async () => { + const fetcher = answering([gitcodePull(ref.source, 'fork/app', 6), gitcodePull('other', 'acme/app', 8), gitcodePull(ref.source)]); + + expect(await lookupOf(new GitCodeProvider()).findOpen(ref)).toBe('https://gitcode.com/acme/app/pull/7'); + const url = new URL(String(fetcher.mock.calls[0]?.[0])); + expect([`${url.origin}${url.pathname}`, url.searchParams.get('state'), url.searchParams.get('head'), url.searchParams.get('base')]) + .toEqual(['https://api.gitcode.com/api/v5/repos/acme/app/pulls', 'open', ref.source, 'main']); + answering([gitcodePull(ref.source, 'fork/app', 6), gitcodePull(ref.source, 'acme/app', 5, 'merged')]); + expect(await lookupOf(new GitCodeProvider()).findOpen(ref)).toBeUndefined(); + }); + + it('fails, rather than finding nothing, on an answer it cannot read or see past, on another host, or with no answer in time', async () => { + const lookup = lookupOf(new GitCodeProvider()); + answering({ message: 'private-token-in-body' }, 401); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitCode answered the open pull request lookup with HTTP 401\.(?!.*private-token-in-body)/); + answering([gitcodePull(ref.source, 'acme/app', 7, 'opened')]); + await expect(lookup.findOpen(ref)).rejects.toThrow(/^GitCode answered the open pull request lookup with data TeamAI cannot read\.$/); + answering(Array.from({ length: 100 }, (_, index) => gitcodePull(`other-${index}`, 'acme/app', index))); + await expect(lookup.findOpen(ref)).rejects.toThrow('more open pull requests than one page'); + answering([gitcodePull(ref.source, 'acme/app', 7, 'open', 'https://evil.example/acme/app/pull/7')]); + await expect(lookup.findOpen(ref)).rejects.toThrow('on another host'); + hanging(); + await expect(lookup.findOpen({ ...ref, timeoutMs: 20 })).rejects.toThrow(); + }); +}); + +describe('the provider of a checkout origin', () => { + it.each([ + ['https://user:secret@github.com/acme/app.git', 'github', 'acme/app'], + ['ssh://git@github.com/acme/app.git', 'github', 'acme/app'], + ['git@github.com:acme/app.git', 'github', 'acme/app'], + ['git@gitlab.com:acme/platform/app.git', 'gitlab', 'acme/platform/app'], + ['https://git.woa.com/acme/app.git', 'tgit', 'acme/app'], + ['https://gitcode.com/acme/app.git', 'gitcode', 'acme/app'], + ])('reads the provider and the repository from %s', (remote, provider, repo) => { + vi.stubEnv('GITLAB_URL', ''); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); + const origin = providerForRemote(remote); + expect(origin.kind === 'provider' && [origin.provider.name, origin.repo]).toEqual([provider, repo]); + }); + + it.each([ + ['https://github.com/acme/app.git', '', 'github.com'], + ['ssh://git@github.com:22/acme/app.git', '', 'github.com'], + ['git@gl.example.com:acme/app.git', 'https://gl.example.com:8443', 'gl.example.com'], + ])('reads the host the pull requests of %s are on', (remote, gitlabUrl, host) => { + vi.stubEnv('GITLAB_URL', gitlabUrl); vi.stubEnv('TEAMAI_GITLAB_HOST', ''); + const origin = providerForRemote(remote); + expect(origin.kind === 'provider' && origin.hostname).toBe(host); + }); + + it.each(['/srv/git/app.git', 'file:///srv/git/app.git', 'https://git.example.internal/acme/app.git', 'https://github.com/acme', 'https://cnb.cool/acme/app.git'])( + 'reaches %s through git alone, which opens no pull request', + (remote) => expect(providerForRemote(remote)).toEqual({ kind: 'transport' }), + ); + + it('a generic git host has no pull request lookup', () => { + expect(new GenericGitProvider().pullRequests).toBeNull(); + }); +}); diff --git a/src/__tests__/workflow-records.test.ts b/src/__tests__/workflow-records.test.ts index b1277bcb6..9939f9c8e 100644 --- a/src/__tests__/workflow-records.test.ts +++ b/src/__tests__/workflow-records.test.ts @@ -23,6 +23,7 @@ import { } from '../workflow/records.js'; import { acquireLock, releaseLock } from '../update.js'; import { generateRunId, isRunId, runLockPath } from '../workflow/store.js'; +import { PrUrlSchema } from '../workflow/pr-url.js'; import type { DriverIdentity, RunState, WorkflowDefinition, WorkflowDiagnostic } from '../workflow/types.js'; const dirs: string[] = []; @@ -204,7 +205,7 @@ describe('#1083 workflow run records', () => { it('publishes the failure code of every step failure teamai writes, and only codes it can name', () => { // Codes a module writes, as `code: '…'`, `error('…', …)` or `failure('…', …)`, that never become a run's failure: every module of src/workflow and src/scorers, and every other one that imports src/scorers or answers a WorkflowEnvelope. - const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'workflow_namespace_conflict', 'prompt_namespace_conflict'] satisfies Array; + const diagnostics = ['unknown_key', 'missing_needs', 'false_if', 'bad_template', 'dotted_name', 'undeclared_output', 'undeclared_end', 'bad_loop', 'bad_workspace', 'workflow_namespace_conflict', 'prompt_namespace_conflict', 'bad_delivery'] satisfies Array; const answers = ['invalid_json', 'driver_running', 'invalid_run', 'repeated_response', 'agent_in_path', 'stale_response', 'retried', 'end_reached', 'repeated_report', 'wrong_attempt', 'invalid_report', 'lease_cancelled', 'interrupted', 'run_unreadable', 'driver_stuck', 'busy', 'workflow_recursion', 'workflow_not_found', // Refusals of a run handover, answered to the command or driver that asked for it. 'not_owner', 'unsupported_repo', 'reports_unreachable', 'reports_failed', 'run_not_found', 'run_not_published', 'run_mismatch', @@ -218,6 +219,8 @@ describe('#1083 workflow run records', () => { 'judge_model_unavailable', // Refusals of a teamai score or teamai workflow command line before it reads any run or judge work. 'invalid_arguments', 'invalid_pass', 'not_initialized', 'unsupported_driver', 'invalid_tools', + // Refusals of teamai workflow start for a run whose pull requests would have no branch to target. + 'no_base_branch', // Refusals of teamai score --run, answered to the member who asked for a pass to be judged. 'scoring_unavailable', 'pass_not_found', 'scorer_not_found', 'entry_leased', // Refusals of teamai score --session, answered to the member who asked for a session to be judged. @@ -234,7 +237,7 @@ describe('#1083 workflow run records', () => { ...[...text.matchAll(/(? match[1] ?? []), ])); // An `error(…)` whose code is not a literal passes on a code written as a literal elsewhere; each one is named here. - const passedOn = ['session-flow/scoring.ts: result.code', 'workflow/engine.ts: tree.failed.code', 'workflow/scoring.ts: built.code', 'workflow/scoring.ts: result.code']; + const passedOn = ['session-flow/scoring.ts: result.code', 'workflow/engine.ts: tree.failed.code', 'workflow/engine.ts: undeliverable.code', 'workflow/scoring.ts: built.code', 'workflow/scoring.ts: result.code']; const unnamed = files.flatMap(({ file, text }) => [...text.matchAll(/(? `${file}: ${match[1]}`)).sort(); const failedWith = (code: string): RunState => runState({ status: 'failed', @@ -394,6 +397,29 @@ describe('#1083 workflow run records', () => { expect(remerged.endedAt).toBe(end.endedAt); }); + it('lists the pull requests delivered on the run in the order they first appeared, and never a step output', () => { + const pr = (n: number): string => `https://github.com/acme/app/pull/${n}`; + const receipt = (n: number, step: string, pass: number) => ({ url: PrUrlSchema.parse(pr(n)), branch: `teamai/r1234567890abcdef/${step}/p${pass}`, base: 'main', step, pass }); + const run = runState({ + steps: [ + { id: 'one', status: 'succeeded', attempt: 'a1', settledAt: 2, outcome: 'success', outputs: { pr_url: 'https://untrusted.example/1' } }, + { id: 'two', status: 'succeeded', attempt: 'a2', settledAt: 3, outcome: 'success' }, + ], + deliveries: [receipt(3, 'two', 1), receipt(1, 'loop.write', 1), receipt(2, 'one', 1)], + }); + + expect(buildEndRecord(run, driver, 'tester').prs).toEqual([pr(3), pr(1), pr(2)]); + expect(buildEndRecord(runState(), driver, 'tester')).not.toHaveProperty('prs'); + }); + + it('drops the prs of an earlier end when a retried run ends again without them', () => { + const delivered = { ...buildEndRecord(runState(), driver, 'tester'), prs: ['https://github.com/acme/app/pull/1'] }; + const again = buildEndRecord(runState(), driver, 'tester'); + + expect(mergeRecord(delivered, again)).not.toHaveProperty('prs'); + expect(mergeRecord(again, delivered).prs).toEqual(['https://github.com/acme/app/pull/1']); + }); + it('queues records durably and keeps them across failure', async () => { const { config } = testConfig(); const record = buildStartRecord(runState({ status: 'running' }), driver, 'tester'); diff --git a/src/__tests__/workflow-scoring.test.ts b/src/__tests__/workflow-scoring.test.ts index a7611b53f..42ecb42c5 100644 --- a/src/__tests__/workflow-scoring.test.ts +++ b/src/__tests__/workflow-scoring.test.ts @@ -16,7 +16,9 @@ import { workflowRunDetail } from '../dashboard/workflow-runs.js'; import { buildEndRecord } from '../workflow/records.js'; import { projectRun } from '../workflow/snapshot.js'; import { nextJudgeLease, parseJudgeRequest, parseJudgeSubmission, runJudgeLease, scoreOnDemand, scoredPasses, submitJudgeLease, type JudgeRequest } from '../workflow/scoring.js'; -import { installOwner, readRun } from '../workflow/store.js'; +import { cancelRun, retryStep } from '../workflow/run-controls.js'; +import { installOwner, readRun, updateRun } from '../workflow/store.js'; +import type { StepState } from '../workflow/step-state.js'; import type { DriverIdentity, RunState, WorkflowEnvelope } from '../workflow/types.js'; import { knownSecrets } from '../env-resolution.js'; @@ -99,7 +101,51 @@ async function settledRun(config: LocalConfig, reported: Record return runId; } +/** Rewrites run `runId`'s one step as `step`, with the run still running, as a delivery would leave it. */ +async function setStep(config: LocalConfig, runId: string, step: (run: RunState) => StepState): Promise { + await updateRun(config, runId, await knownSecrets(config), async (run) => (run === 'missing' ? { value: undefined } : { value: undefined, write: { ...run, status: 'running', steps: [step(run)] } })); +} + describe('judge work', () => { + it('judges a step while it delivers, and its label stays the judgement however the delivery ends', async () => { + const config = fixture(scorer(100)); + const runId = await settledRun(config); + const settled = (await runOf(config, runId)).steps[0]; + if (settled?.status !== 'succeeded') throw new Error('review did not succeed'); + const reported = (run: RunState) => ({ ...settled, delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } }); + await setStep(config, runId, (run) => ({ ...reported(run), status: 'delivering' })); + + const lease = leaseOf(await nextJudgeLease(config, request())); + data(await submitJudgeLease(config, lease.ref, { driver: judgeDriver, outcome: 'success', answer: { label: 'good', reasoning: 'r' } })); + const labels = await LabelStore.local(config); + expect(labels).toMatchObject([{ subject: { runId, step: 'review', pass: 1 }, label: 'good' }]); + + await setStep(config, runId, (run) => ({ ...reported(run), status: 'failed', outcome: 'failed', failure: { code: 'delivery_failed', message: 'm' } })); + expect(data(await nextJudgeLease(config, request()))).not.toHaveProperty('lease'); + await setStep(config, runId, (run) => ({ ...reported(run), status: 'succeeded', outcome: 'success' })); + expect(data(await nextJudgeLease(config, request()))).not.toHaveProperty('lease'); + expect(await LabelStore.local(config)).toEqual(labels); + }); + + it('judges a step cancelled while it delivers as its agent left it, and the run record keeps the tokens of that pass', async () => { + const config = fixture(scorer(100)); + const tokens = { input: 1200, output: 80 }; + const runId = await settledRun(config, { tokens }); + const settled = (await runOf(config, runId)).steps[0]; + if (settled?.status !== 'succeeded') throw new Error('review did not succeed'); + await setStep(config, runId, (run) => ({ ...settled, status: 'delivering', delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } })); + expect(await cancelRun(config, runId, await knownSecrets(config), 'review')).toMatchObject({ data: { cancelled: { steps: ['review'] } } }); + + const cancelled = await runOf(config, runId); + expect(cancelled.steps[0]).toMatchObject({ status: 'cancelled', tokens, outputs: { reviewed: true } }); + expect(buildEndRecord(cancelled, workDriver, 'tester').steps).toEqual([expect.objectContaining({ step: 'review', pass: 1, outcome: 'cancelled', tokens })]); + expect(data(await nextJudgeLease(config, request()))).toHaveProperty('lease'); + + expect(await retryStep(config, runId, await knownSecrets(config), 'review')).toMatchObject({ data: { retried: { step: 'review', pass: 2 } } }); + const install = (await installOwner(config)).id; + expect(scoredPasses(await runOf(config, runId), install)).toEqual([{ subject: { runId, step: 'review', pass: 1 }, scorers: ['quality'] }]); + }); + it('finalizes the selected Codex tool in the lease', async () => { const config = fixture(scorer(100)); await settledRun(config); @@ -546,6 +592,28 @@ describe('scoring totals', () => { }); describe('scoredPasses', () => { + it('judges the agent pass of a step that delivers, for the install that reported it, whatever becomes of its delivery', async () => { + const config = fixture(scorer(100)); + const run = await runOf(config, await settledRun(config)); + const install = (await installOwner(config)).id; + const reported = { id: 'review', settledAt: 2, attempt: 'a1', installId: install, delivery: { branch: `teamai/${run.id}/review/p1`, head: 'c'.repeat(40) } }; + const judged = [{ subject: { runId: run.id, step: 'review', pass: 1 }, scorers: ['quality'] }]; + const withStep = (step: StepState): RunState => ({ ...run, status: 'running', steps: [step] }); + + expect(scoredPasses(withStep({ ...reported, status: 'delivering' }), install)).toEqual(judged); + expect(scoredPasses(withStep({ ...reported, status: 'succeeded', outcome: 'success' }), install)).toEqual(judged); + for (const code of ['delivery_failed', 'protected_path']) { + expect(scoredPasses(withStep({ ...reported, status: 'failed', outcome: 'failed', failure: { code, message: 'm' } }), install)).toEqual(judged); + const retried: StepState = { id: 'review', status: 'pending', pass: 2, passes: [{ pass: 1, outcome: 'failed', failure: { code, message: 'm' }, installId: install }] }; + expect(scoredPasses(withStep(retried), install)).toEqual(judged); + } + expect(scoredPasses(withStep({ id: 'review', status: 'failed', outcome: 'failed', settledAt: 2, installId: install, failure: { code: 'tool_failed', message: 'm' } }), install)).toEqual([]); + // Claimed mid-delivery: the install that reported the pass judges it, the one that claimed the run does not. + const claimed = { ...withStep({ ...reported, status: 'delivering' }), publication: { stateRevision: 2 } }; + expect(scoredPasses(claimed, install)).toEqual(judged); + expect(scoredPasses(claimed, 'claimer')).toEqual([]); + }); + it('keeps only the passes this install settled, and attributes archived children to the workflow pass that held them', async () => { const config = fixture(scorer(100)); const run = await runOf(config, await settledRun(config)); diff --git a/src/__tests__/workflow-step-state.test.ts b/src/__tests__/workflow-step-state.test.ts new file mode 100644 index 000000000..4d054f912 --- /dev/null +++ b/src/__tests__/workflow-step-state.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, expectTypeOf, it } from 'vitest'; +import { + StepCheckpointSchema, StepStateSchema, cancellable, inProgress, type StepCheckpoint, type StepState, +} from '../workflow/step-state.js'; + +function statusesOf(schema: typeof StepStateSchema | typeof StepCheckpointSchema): string[] { + return schema.options.map((option) => option.shape.status.value).sort(); +} + +const step = (status: StepState['status']): StepState => StepStateSchema.parse({ + id: 'work', status, + ...(status === 'leased' ? { attempt: 'a1', lease: { attempt: 'a1', revision: 1, driver: { kind: 'local', machine: 'm', workspace: 'w' } } } : {}), + ...(status === 'cancelled' ? { failure: { code: 'cancelled', message: 'Cancelled' } } : {}), + ...(status === 'delivering' ? { delivery: { branch: 'teamai/r1/work/p1', head: 'c'.repeat(40) } } : {}), +}); + +describe('step statuses', () => { + it('a handed-over run carries every status a local run has', () => { + expectTypeOf().toEqualTypeOf(); + expect(statusesOf(StepCheckpointSchema)).toEqual(statusesOf(StepStateSchema)); + }); + + it.each([ + ['pending', false, true], ['waiting', true, true], ['leased', true, true], ['running', true, true], + ['delivering', true, true], ['succeeded', false, false], ['failed', false, false], ['skipped', false, false], + ['cancelled', false, false], + ] as const)('a %s step is in progress: %s, and can be cancelled: %s', (status, open, stoppable) => { + expect(inProgress(step(status))).toBe(open); + expect(cancellable(step(status))).toBe(stoppable); + }); +}); diff --git a/src/dashboard/client.ts b/src/dashboard/client.ts index 11cba0357..bfce94e0e 100644 --- a/src/dashboard/client.ts +++ b/src/dashboard/client.ts @@ -188,7 +188,7 @@ async function saveSelection() { if(!dist||!distDraft||!preview)return; const ge // records the team published. Rows carry only run-record fields, and a HITL // answer posts the pass the page showed so a replaced request is refused. const runStatusNames = { running:'Running', succeeded:'Succeeded', failed:'Failed', cancelled:'Cancelled' }; -const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; +const stepStatusNames = { pending:'Pending', waiting:'Waiting for an answer', leased:'Leased', running:'Running', delivering:'Delivering PR', succeeded:'Succeeded', failed:'Failed', skipped:'Skipped', cancelled:'Cancelled' }; function runStatus(status) { return ''+label(runStatusNames[status]||stepStatusNames[status]||status)+''; } // A running run's duration is filled by tickRunDurations, so a poll that changes nothing leaves the HTML, and the buttons in it, alone. function runDurationHtml(r) { return r.durationMs==null&&r.status==='running'?'':e(elapsed(r.durationMs ?? NaN)); } diff --git a/src/dashboard/locales.ts b/src/dashboard/locales.ts index a4174bdd3..411191896 100644 --- a/src/dashboard/locales.ts +++ b/src/dashboard/locales.ts @@ -264,6 +264,7 @@ export const dashboardMessages: Record = { "Cancelled": "已取消", "Pending": "待运行", "Leased": "已租用", + "Delivering PR": "正在交付 PR", "Skipped": "已跳过", "Waiting for an answer": "等待回答", "Waiting": "等待中", diff --git a/src/dashboard/workflow-runs.ts b/src/dashboard/workflow-runs.ts index bd02bff89..7b43a4df2 100644 --- a/src/dashboard/workflow-runs.ts +++ b/src/dashboard/workflow-runs.ts @@ -93,7 +93,7 @@ interface RunRow { /** A run on this machine, as its run file stands now. */ export type LocalRunRow = RunRow & { source: 'local'; - /** The leased steps: a driver may work on several at once. */ + /** Agent leases and deliveries currently in progress. */ current: string[]; /** The steps whose HITL request waits for an answer. */ waiting: string[]; @@ -177,7 +177,7 @@ function localRow(run: RunState, records: Map): LocalRunRow return { ...rowOf(localRecord(run)), source: 'local', - current: run.steps.filter((state) => state.status === 'leased').map((state) => state.id), + current: run.steps.filter((state) => state.status === 'leased' || state.status === 'delivering').map((state) => state.id), waiting: run.steps.filter((state) => state.status === 'waiting').map((state) => state.id), ...(record ? { record } : {}), }; diff --git a/src/index.ts b/src/index.ts index 6130b18f4..a603d77b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -796,7 +796,7 @@ envCmd const workflowCmd = program .command('workflow') - .description('Run a validated workflow from workflows/.yaml'); + .description('Run a validated workflow from workflows/.yaml. Writable steps may set deliver: pr to open review requests; TeamAI never merges them.'); workflowCmd .command('start ') @@ -841,7 +841,7 @@ workflowCmd workflowCmd .command('retry ') - .description('Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope.') + .description('Retry a leased or stopped step, or a HITL request to answer it again, with a new pass and revision. A failed pull request delivery is retried in the same pass, without running the step again. Reopen its dependent steps and a settled run; keep earlier passes. After reopening, run resume or let your driver poll next. Prints one JSON envelope.') .action(async (run: string, step: string) => { const { workflowRetry } = await import('./workflow-commands.js'); await workflowRetry(run, step); @@ -869,7 +869,7 @@ workflowCmd workflowCmd .command('cancel [step]') - .description('Cancel a whole run, or a pending, waiting, leased, or running step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope.') + .description('Cancel a whole run, or a pending, waiting, leased, running, or delivering step and its open descendants. Invalidates leases; drivers stop tools listed in next.cancelled. Prints one JSON envelope.') .action(async (run: string, step?: string) => { const { workflowCancel } = await import('./workflow-commands.js'); await workflowCancel(run, step); diff --git a/src/providers/cnb/cnb-cli.ts b/src/providers/cnb/cnb-cli.ts index dc85b8d00..43e24785c 100644 --- a/src/providers/cnb/cnb-cli.ts +++ b/src/providers/cnb/cnb-cli.ts @@ -43,7 +43,7 @@ export const CNB_HOST = process.env.TEAMAI_CNB_HOST?.trim() || 'cnb.cool'; /** Run a `cnb` subcommand. Returns { stdout, stderr, status }. */ export function cnbExec( args: string[], - options?: { inheritStdio?: boolean; cwd?: string }, + options?: { inheritStdio?: boolean; cwd?: string; timeoutMs?: number }, ): { stdout: string; stderr: string; status: number } { // Resolve the executable first, then launch it through cross-spawn: on // Windows the npm-installed CLI is only a `.cmd` shim, which neither a bare @@ -58,7 +58,7 @@ export function cnbExec( log.debug(`cnb exec: ${cnbPath} ${args.join(' ')}`); if (options?.inheritStdio) { - const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd }); + const r = crossSpawn.sync(cnbPath, args, { stdio: 'inherit', env: { ...process.env }, cwd: options.cwd, ...(options.timeoutMs ? { timeout: options.timeoutMs } : {}) }); return { stdout: '', stderr: '', status: r.status ?? 1 }; } const r = crossSpawn.sync(cnbPath, args, { @@ -66,6 +66,7 @@ export function cnbExec( encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024, cwd: options?.cwd, + ...(options?.timeoutMs ? { timeout: options.timeoutMs } : {}), }); return { stdout: (r.stdout ?? '').toString().trim(), @@ -339,6 +340,7 @@ export interface CnbPullCreateOptions { title: string; description?: string; cwd?: string; + spawnTimeoutMs?: number; } /** @@ -357,7 +359,7 @@ export function cnbPullCreate(opts: CnbPullCreateOptions): string { ]; if (opts.description) args.push('--body', opts.description); - const r = cnbExec(args, { cwd: opts.cwd }); + const r = cnbExec(args, { cwd: opts.cwd, ...(opts.spawnTimeoutMs ? { timeoutMs: opts.spawnTimeoutMs } : {}) }); if (r.status !== 0) { throw new Error(`cnb post-pull failed: ${r.stderr || r.stdout}`); } diff --git a/src/providers/cnb/index.ts b/src/providers/cnb/index.ts index b3a3a02a9..52b718a66 100644 --- a/src/providers/cnb/index.ts +++ b/src/providers/cnb/index.ts @@ -20,6 +20,12 @@ import { */ export class CNBProvider implements GitProvider { readonly name = 'cnb'; + /** + * None, so workflow delivery pushes to a CNB origin and opens no pull request. Delivery must find an open request + * before it opens one, and neither this code nor its docs confirm how the `cnb` CLI lists open pull requests (even the + * `post-pull` output `cnbPullCreate` parses is unconfirmed), so a lookup could answer a false "none" and open a second. + */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return cnbParseRepoInput(input); @@ -72,6 +78,7 @@ export class CNBProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/git/index.ts b/src/providers/git/index.ts index 200b79ced..706809bbe 100644 --- a/src/providers/git/index.ts +++ b/src/providers/git/index.ts @@ -57,6 +57,8 @@ function gitIdentity(): string { */ export class GenericGitProvider implements GitProvider { readonly name = 'git'; + /** Git alone opens no pull request. */ + readonly pullRequests = null; parseRepoInput(input: string): RepoInfo { return parseGenericGitRepoInput(input); diff --git a/src/providers/gitcode/gitcode-api.ts b/src/providers/gitcode/gitcode-api.ts index b4ea6da60..37cf7ec41 100644 --- a/src/providers/gitcode/gitcode-api.ts +++ b/src/providers/gitcode/gitcode-api.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { z } from 'zod'; +import type { PullRequestRef } from '../types.js'; import { GITCODE_HOST } from './repo-url.js'; import { getUserHome, legacyUserHome } from '../../utils/home.js'; import { sanitizeGitUrl } from '../../utils/redact.js'; @@ -274,6 +276,7 @@ export interface GitCodePullCreateOptions { description?: string; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } /** @@ -296,6 +299,7 @@ export async function gitcodePullCreate(opts: GitCodePullCreateOptions): Promise headers: authHeaders(token), body: JSON.stringify(body), redirect: 'manual', + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), }); if (!resp.ok) { const errBody = await resp.text().catch(() => ''); @@ -309,3 +313,41 @@ export async function gitcodePullCreate(opts: GitCodePullCreateOptions): Promise if (pr.number) return `https://${GITCODE_HOST}/${opts.repo}/pull/${pr.number}`; throw new Error('GitCode PR created but response did not include html_url.'); } + +// ─── Open pull request lookup ──────────────────────────── + +const LOOKUP_PAGE = 100; + +/** A listed pull request; a state outside Gitee's three is one this cannot read. */ +const GitCodeOpenPullSchema = z.object({ + html_url: z.string(), + state: z.enum(['open', 'closed', 'merged']), + head: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }).nullable() }), + base: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }) }), +}); + +/** + * The open pull request from `source` into `target` in `repo`, asked of the GitCode REST API with the token + * `getGitCodeToken` resolves. A request from a fork with the same branch name is not it. Never prompts, and throws, + * rather than answering none, when GitCode cannot be asked or answers in time, answers what this cannot read, fills a + * whole page without it, or names a request on another host. + */ +export async function gitcodeFindOpenPull({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const token = requireToken(); + const query = new URLSearchParams({ state: 'open', head: source, base: target, per_page: String(LOOKUP_PAGE) }); + const resp = await fetch(`${gitcodeApiBase()}/repos/${repo}/pulls?${query}`, { + headers: authHeaders(token), redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!resp.ok) throw new Error(`GitCode answered the open pull request lookup with HTTP ${resp.status}. Check GITCODE_TOKEN and its access to the repository.`); + const pulls = z.array(GitCodeOpenPullSchema).safeParse(await resp.json().catch(() => undefined)); + if (!pulls.success) throw new Error('GitCode answered the open pull request lookup with data TeamAI cannot read.'); + const ours = (name: string | undefined): boolean => name?.toLowerCase() === repo.toLowerCase(); + const found = pulls.data.find((pull) => pull.state === 'open' && pull.head.ref === source && pull.base.ref === target + && ours(pull.head.repo?.full_name) && ours(pull.base.repo.full_name)); + if (!found) { + if (pulls.data.length >= LOOKUP_PAGE) throw new Error('GitCode listed more open pull requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (!found.html_url.startsWith(`https://${GITCODE_HOST}/`)) throw new Error('GitCode answered the open pull request lookup with a URL on another host.'); + return found.html_url; +} diff --git a/src/providers/gitcode/index.ts b/src/providers/gitcode/index.ts index 88ec983d2..09360873e 100644 --- a/src/providers/gitcode/index.ts +++ b/src/providers/gitcode/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { gitcodeIsAuthenticated, @@ -6,6 +6,7 @@ import { gitcodeRepoClone, gitcodeCreateRepo, gitcodePullCreate, + gitcodeFindOpenPull, writeNetrcToken, GitCodeRepoNotFoundError, } from './gitcode-api.js'; @@ -26,6 +27,7 @@ import { askQuestion, isInteractive } from '../../utils/prompt.js'; */ export class GitCodeProvider implements GitProvider { readonly name = 'gitcode'; + readonly pullRequests: PullRequestLookup = { findOpen: gitcodeFindOpenPull }; parseRepoInput(input: string): RepoInfo { return parseGitCodeRepoInput(input); @@ -111,6 +113,7 @@ export class GitCodeProvider implements GitProvider { title: opts.title, description: opts.description, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/github/gh-cli.ts b/src/providers/github/gh-cli.ts index e7a1c49c4..c929c11b4 100644 --- a/src/providers/github/gh-cli.ts +++ b/src/providers/github/gh-cli.ts @@ -1,8 +1,10 @@ import { spawnSync } from 'node:child_process'; import crossSpawn from 'cross-spawn'; +import { z } from 'zod'; import { log } from '../../utils/logger.js'; import { resolveCliPath } from '../../utils/cli-path.js'; import { isInteractive } from '../../utils/prompt.js'; +import type { PullRequestRef } from '../types.js'; // ─── Constants ─────────────────────────────────────────── @@ -562,6 +564,7 @@ async function ghPrCreateViaApi(opts: GhPrCreateOptions): Promise { { method: 'POST', headers: authHeaders, + ...(opts.spawnTimeoutMs ? { signal: AbortSignal.timeout(opts.spawnTimeoutMs) } : {}), body: JSON.stringify({ title: opts.title, body: opts.description ?? opts.title, @@ -599,3 +602,84 @@ async function ghPrCreateViaApi(opts: GhPrCreateOptions): Promise { } return pr.html_url; } + +// ─── Open pull request lookup ──────────────────────────── + +const GhPullSchema = z.object({ + html_url: z.string(), + state: z.string(), + head: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }).nullable() }), + base: z.object({ ref: z.string(), repo: z.object({ full_name: z.string() }) }), +}); + +/** + * The open pull request from `source` into `target` in `repo`, asked of an installed gh or, without one, of the API with + * GITHUB_TOKEN or GH_TOKEN. A request from a fork with the same branch name is not it. Never logs in or prompts, and + * throws, rather than answering none, when GitHub cannot be asked or answers what this cannot read. + */ +export async function ghFindOpenPull({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const owner = repo.split('/')[0] ?? repo; + const query = new URLSearchParams({ state: 'open', head: `${owner}:${source}`, base: target, per_page: '100' }); + const pulls = z.array(GhPullSchema).safeParse(await ghApiGet(`repos/${repo}/pulls?${query}`, timeoutMs)); + if (!pulls.success) throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + const ours = (name: string | undefined): boolean => name?.toLowerCase() === repo.toLowerCase(); + return pulls.data.find((pull) => pull.state === 'open' && pull.head.ref === source && pull.base.ref === target + && ours(pull.head.repo?.full_name) && ours(pull.base.repo.full_name))?.html_url; +} + +async function ghApiGet(path: string, timeoutMs: number): Promise { + const ghPath = getGhPath(); + if (ghPath) { + const result = await ghSpawn(ghPath, ['api', path], { timeoutMs, env: { GH_PROMPT_DISABLED: '1', GIT_TERMINAL_PROMPT: '0' } }); + if (result.status !== 0) throw new Error('gh could not look up open pull requests. Check gh auth status and its access to the repository.'); + try { + return JSON.parse(result.stdout); + } catch { + throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + } + } + const token = getGitHubToken(); + if (!token) throw new Error('GitHub credentials are unavailable: install gh and log in, or set GITHUB_TOKEN or GH_TOKEN.'); + const response = await fetch(`${GITHUB_API}/${path}`, { + headers: { 'Accept': 'application/vnd.github+json', 'Authorization': `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28' }, + redirect: 'manual', + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`GitHub answered the open pull request lookup with HTTP ${response.status}. Check the token's access to the repository.`); + try { + return await response.json(); + } catch { + throw new Error('GitHub answered the open pull request lookup with data TeamAI cannot read.'); + } +} + +/** + * Runs gh as `ghExec` does, through cross-spawn with the same arguments, environment and timeout, but without blocking + * the process, which a workflow driver shares with the agents it runs, while gh waits on GitHub. + */ +function ghSpawn(ghPath: string, args: string[], { env, timeoutMs }: { env: NodeJS.ProcessEnv; timeoutMs: number }): Promise<{ stdout: string; status: number }> { + log.debug(`gh exec: ${ghPath} ${args.join(' ')}`); + return new Promise((resolve) => { + const child = crossSpawn(ghPath, args, { env: { ...process.env, ...env } }); + let stdout = ''; + let settled = false; + const finish = (status: number): void => { + if (settled) return; + settled = true; + clearTimeout(deadline); + resolve({ stdout: stdout.trim(), status }); + }; + const deadline = setTimeout(() => { + finish(1); + child.kill('SIGKILL'); + child.stdin?.destroy(); + child.stdout?.destroy(); + child.stderr?.destroy(); + }, timeoutMs); + child.stdout?.setEncoding('utf8'); + child.stdout?.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr?.resume(); + child.on('error', () => finish(1)); + child.on('close', (code) => finish(code ?? 1)); + }); +} diff --git a/src/providers/github/index.ts b/src/providers/github/index.ts index 9d7af0342..4ef461626 100644 --- a/src/providers/github/index.ts +++ b/src/providers/github/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGhAvailable, @@ -8,6 +8,7 @@ import { ghRepoClone, ghCreateRepo, ghPrCreate, + ghFindOpenPull, RepoNotFoundError as GhRepoNotFoundError, } from './gh-cli.js'; import { ghListOrgRepos } from './gh-org.js'; @@ -15,6 +16,7 @@ import { parseGitHubRepoInput } from './repo-url.js'; export class GitHubProvider implements GitProvider { readonly name = 'github'; + readonly pullRequests: PullRequestLookup = { findOpen: ghFindOpenPull }; parseRepoInput(input: string): RepoInfo { return parseGitHubRepoInput(input); diff --git a/src/providers/gitlab/gitlab-api.ts b/src/providers/gitlab/gitlab-api.ts index 9409bf0d5..d5e252d8c 100644 --- a/src/providers/gitlab/gitlab-api.ts +++ b/src/providers/gitlab/gitlab-api.ts @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process'; +import { z } from 'zod'; import { GITLAB_HOST } from './repo-url.js'; import { sanitizeGitUrl } from '../../utils/redact.js'; +import type { PullRequestRef } from '../types.js'; /** * GitLab REST API client (API v4). @@ -301,6 +303,7 @@ export interface GitLabMrCreateOptions { reviewers?: string[]; /** Working directory (unused for the REST path, kept for interface parity) */ cwd?: string; + spawnTimeoutMs?: number; } interface GitLabUser { @@ -355,6 +358,7 @@ export async function gitlabMrCreate(opts: GitLabMrCreateOptions): Promise ''); @@ -366,3 +370,49 @@ export async function gitlabMrCreate(opts: GitLabMrCreateOptions): Promise { + const token = requireToken(); + const query = new URLSearchParams({ state: 'opened', source_branch: source, target_branch: target, per_page: String(LOOKUP_PAGE) }); + const response = await fetch(`${gitlabApiBase()}/projects/${encodeURIComponent(repo)}/merge_requests?${query}`, { + headers: authHeaders(token), redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`GitLab answered the open merge request lookup with HTTP ${response.status}. Check the token's access to the project.`); + const requests = z.array(GitLabOpenMrSchema).safeParse(await response.json().catch(() => undefined)); + if (!requests.success) throw new Error('GitLab answered the open merge request lookup with data TeamAI cannot read.'); + const found = requests.data.find((mr) => mr.state === 'opened' && mr.source_branch === source && mr.target_branch === target + && mr.source_project_id === mr.target_project_id); + if (!found) { + if (requests.data.length >= LOOKUP_PAGE) throw new Error('GitLab listed more open merge requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (hostOf(found.web_url) !== hostOf(gitlabBaseUrl())) throw new Error('GitLab answered the open merge request lookup with a URL on another host.'); + return found.web_url; +} + +function hostOf(url: string): string | undefined { + try { + return new URL(url).host; + } catch { + return undefined; + } +} diff --git a/src/providers/gitlab/index.ts b/src/providers/gitlab/index.ts index 506a810a5..69defc2cf 100644 --- a/src/providers/gitlab/index.ts +++ b/src/providers/gitlab/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGitLabAvailable, @@ -7,6 +7,7 @@ import { gitlabRepoClone, gitlabCreateRepo, gitlabMrCreate, + gitlabFindOpenMr, GitLabRepoNotFoundError, } from './gitlab-api.js'; import { gitlabListOrgRepos } from './org.js'; @@ -22,6 +23,7 @@ import type { MRData } from '../../types.js'; */ export class GitLabProvider implements GitProvider { readonly name = 'gitlab'; + readonly pullRequests: PullRequestLookup = { findOpen: gitlabFindOpenMr }; parseRepoInput(input: string): RepoInfo { return parseGitLabRepoInput(input); @@ -72,6 +74,7 @@ export class GitLabProvider implements GitProvider { description: opts.description, reviewers: opts.reviewers, cwd: opts.cwd, + spawnTimeoutMs: opts.spawnTimeoutMs, }); } diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 36057530f..c054c66fe 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1,4 +1,4 @@ -import type { GitProvider } from './types.js'; +import type { GitProvider, PullRequestLookup } from './types.js'; import { TGitProvider } from './tgit/index.js'; import { GitHubProvider } from './github/index.js'; import { CNBProvider } from './cnb/index.js'; @@ -213,3 +213,54 @@ export function getProviderFromUrl(repoUrl: string): GitProvider { const name = detectProvider(repoUrl); return getProvider(name); } + +// ─── Workflow delivery ─────────────────────────────────── + +/** + * A checkout's origin as workflow delivery reaches it: through a provider that can look up its pull requests, or git + * alone. `hostname` is the host the origin's pull requests are on: its web host, which an ssh or scp remote shares, since + * a self-hosted GitLab refuses a GITLAB_URL on another host than TEAMAI_GITLAB_HOST. + */ +export type OriginProvider = + | { kind: 'provider'; provider: GitProvider; pullRequests: PullRequestLookup; repo: string; hostname: string } + | { kind: 'transport' }; + +/** + * The provider of a checkout's configured origin URL, `remote.origin.url` as written before `insteadOf` rewrites it, and + * the repository on it. Credentials, query and fragment are dropped first. A filesystem path, a file:// URL, a host no + * provider owns, a provider with no pull request lookup, or a URL its provider cannot parse is reached through git alone. + * The bare `owner/repo` default is for user input, never for a remote. + */ +export function providerForRemote(remoteUrl: string): OriginProvider { + const hosted = hostingUrl(remoteUrl.trim()); + if (!hosted) return { kind: 'transport' }; + const provider = getProviderFromUrl(hosted.href); + if (!provider.pullRequests) return { kind: 'transport' }; + try { + const { owner, repo } = provider.parseRepoInput(hosted.href); + return { kind: 'provider', provider, pullRequests: provider.pullRequests, repo: `${owner}/${repo}`, hostname: hosted.hostname }; + } catch { + return { kind: 'transport' }; + } +} + +/** The web URL of a remote in its https, ssh or scp form, without credentials, query or fragment; undefined for any other form. */ +function hostingUrl(remote: string): URL | undefined { + const scp = /^[^@\s/]+@([^:\s/]+):(.+)$/.exec(remote); + if (scp) { + const [, host, repoPath] = scp; + return host && repoPath ? parsedUrl(`https://${host}/${repoPath}`) : undefined; + } + if (!/^(https?|ssh):\/\//i.test(remote)) return undefined; + const url = parsedUrl(remote); + if (!url) return undefined; + return parsedUrl(url.protocol === 'ssh:' ? `https://${url.hostname}${url.pathname}` : `${url.protocol}//${url.host}${url.pathname}`); +} + +function parsedUrl(value: string): URL | undefined { + try { + return new URL(value); + } catch { + return undefined; + } +} diff --git a/src/providers/tgit/gf-cli.ts b/src/providers/tgit/gf-cli.ts index 3f928b80a..822440f6e 100644 --- a/src/providers/tgit/gf-cli.ts +++ b/src/providers/tgit/gf-cli.ts @@ -8,7 +8,10 @@ import { getUserHome, legacyUserHome } from '../../utils/home.js'; import { log, spinner } from '../../utils/logger.js'; import { isInteractive } from '../../utils/prompt.js'; import { getTeamaiHomeDir } from '../../types.js'; +import { z } from 'zod'; +import type { PullRequestRef } from '../types.js'; import { tgitFetch, tgitGitCloneUrl } from './rest-auth.js'; +import { TGIT_HOST } from './repo-url.js'; /** Path where gf CLI is installed */ function gfInstallDir(): string { @@ -552,3 +555,40 @@ export function gfMrCreate(opts: GfMrCreateOptions): string { // Could not extract MR URL — treat as failure throw new Error(`gf mr create succeeded but returned unexpected output: ${output}`); } + +// ─── Open merge request lookup ─────────────────────────── + +const LOOKUP_PAGE = 100; + +const TGitOpenMrSchema = z.object({ + web_url: z.string(), + state: z.string(), + source_branch: z.string(), + target_branch: z.string(), + source_project_id: z.number(), + target_project_id: z.number(), +}); + +/** + * The open merge request from `source` into `target` in project `repo`, asked of the git.woa.com REST API with the + * credential `tgitFetch` resolves; a reopened request is open too. A request from a fork with the same branch name is + * not it. Never prompts, and throws, rather than answering none, when TGit cannot be asked or answers in time, answers + * what this cannot read, fills a whole page without it, or names a request on another host. + */ +export async function tgitFindOpenMr({ repo, source, target, timeoutMs }: PullRequestRef): Promise { + const query = new URLSearchParams({ state: 'opened', source_branch: source, target_branch: target, per_page: String(LOOKUP_PAGE) }); + const response = await tgitFetch(`/projects/${encodeURIComponent(repo)}/merge_requests?${query}`, { + redirect: 'manual', signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) throw new Error(`TGit answered the open merge request lookup with HTTP ${response.status}. Check TGIT_TOKEN or the gf login and their access to the project.`); + const requests = z.array(TGitOpenMrSchema).safeParse(await response.json().catch(() => undefined)); + if (!requests.success) throw new Error('TGit answered the open merge request lookup with data TeamAI cannot read.'); + const found = requests.data.find((mr) => (mr.state === 'opened' || mr.state === 'reopened') && mr.source_branch === source + && mr.target_branch === target && mr.source_project_id === mr.target_project_id); + if (!found) { + if (requests.data.length >= LOOKUP_PAGE) throw new Error('TGit listed more open merge requests than one page, so TeamAI cannot tell whether this branch has one.'); + return undefined; + } + if (!found.web_url.startsWith(`https://${TGIT_HOST}/`)) throw new Error('TGit answered the open merge request lookup with a URL on another host.'); + return found.web_url; +} diff --git a/src/providers/tgit/index.ts b/src/providers/tgit/index.ts index da5d724f0..3534e897e 100644 --- a/src/providers/tgit/index.ts +++ b/src/providers/tgit/index.ts @@ -1,4 +1,4 @@ -import type { GitProvider, PrCreateOptions, RepoInfo, OrgRepoInfo } from '../types.js'; +import type { GitProvider, PrCreateOptions, PullRequestLookup, RepoInfo, OrgRepoInfo } from '../types.js'; import { RepoNotFoundError } from '../types.js'; import { ensureGfInstalled, @@ -8,6 +8,7 @@ import { gfRepoClone, gfCreateRepo, gfMrCreate, + tgitFindOpenMr, RepoNotFoundError as GfRepoNotFoundError, } from './gf-cli.js'; import { gfListOrgRepos } from './gf-org.js'; @@ -15,6 +16,7 @@ import { parseTGitRepoInput } from './repo-url.js'; export class TGitProvider implements GitProvider { readonly name = 'tgit'; + readonly pullRequests: PullRequestLookup = { findOpen: tgitFindOpenMr }; parseRepoInput(input: string): RepoInfo { return parseTGitRepoInput(input); diff --git a/src/providers/tgit/repo-url.ts b/src/providers/tgit/repo-url.ts index cf87b9ff3..7a3b408ae 100644 --- a/src/providers/tgit/repo-url.ts +++ b/src/providers/tgit/repo-url.ts @@ -1,6 +1,6 @@ import type { RepoInfo } from '../types.js'; -const TGIT_HOST = 'git.woa.com'; +export const TGIT_HOST = 'git.woa.com'; /** * Parse user input into a standardized RepoInfo structure for TGit. diff --git a/src/providers/types.ts b/src/providers/types.ts index ccf95ab8b..166569c95 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -48,6 +48,25 @@ export interface PrCreateOptions { spawnTimeoutMs?: number; } +/** The pull request workflow delivery looks for: head branch `source` into `target`, both in `repo`. */ +export interface PullRequestRef { + /** Repository in "owner/repo" format, as parseRepoInput gives it. */ + repo: string; + source: string; + target: string; + /** How long the host may take to answer. */ + timeoutMs: number; +} + +/** + * Read-only pull request lookup for workflow delivery. It has no merge, close, approve or update member, and a test + * pins its key set. It never prompts, logs in or calls authenticate(). + */ +export interface PullRequestLookup { + /** The web URL of the open request `ref` names, if any. Throws when the host cannot answer, never answering "none". */ + findOpen(ref: PullRequestRef): Promise; +} + /** * 轻量级仓库元信息，用于 listOrgRepos 返回。 */ @@ -120,6 +139,12 @@ export interface GitProvider { */ createPullRequest(opts: PrCreateOptions): Promise; + /** + * How workflow delivery finds the open request of a branch before it opens one, so a delivery that ran before opens + * no second request. Null for a host only git reaches, where delivery opens none. + */ + readonly pullRequests: PullRequestLookup | null; + /** * 获取指定 MR/PR 的完整数据（标题、描述、提交列表、diff）。 * diff --git a/src/utils/git-env.ts b/src/utils/git-env.ts index 6c10f7c3a..cf0f3719f 100644 --- a/src/utils/git-env.ts +++ b/src/utils/git-env.ts @@ -23,10 +23,11 @@ import { isInteractive } from './prompt.js'; /** - * Applied only where the caller left the variable unset, so an explicit value - * always wins. + * Applied process-wide only where the caller left the variable unset, so an + * explicit value always wins there. A git child that must never prompt, even + * in an interactive run, gets it as its own environment. */ -const NON_INTERACTIVE_GIT_ENV: Record = { +export const NON_INTERACTIVE_GIT_ENV: Readonly> = { GIT_TERMINAL_PROMPT: '0', // `echo` answers the askpass request with an empty line, so the credential is // rejected at once instead of a dialog waiting for a person who is not there. diff --git a/src/workflow/conformance-interpreter.ts b/src/workflow/conformance-interpreter.ts index c4c9bf812..19645916e 100644 --- a/src/workflow/conformance-interpreter.ts +++ b/src/workflow/conformance-interpreter.ts @@ -1,4 +1,5 @@ /** The conformance scenario language, the driver profiles it speaks as, and the interpreter that runs a scenario. */ +import { z } from 'zod'; import type { DriverIdentity, RunState, WorkflowEnvelope } from './types.js'; export interface WorkflowClient { @@ -87,10 +88,23 @@ export type ConformanceOperation = as?: DriverProfile; attempt?: string; revision?: number; + /** Fields added to the `changes` the harness reports from the workspace of this operation's saved lease. */ + changes?: Record; body: Record; expect: Record; }; +const SavedLeaseSchema = z.object({ + attempt: z.string(), revision: z.number(), workspace: z.object({ branch: z.string(), base: z.object({ commit: z.string() }) }).optional(), +}); +type SavedLease = z.infer; + +/** The `changes` a report of `lease` carries: the branch and base of its workspace, and the scenario's own fields. */ +function leasedChanges(lease: SavedLease | undefined, fields: Record): Record { + if (!lease?.workspace) throw new Error('A report with changes needs a saved lease that ran in a workspace'); + return { branch: lease.workspace.branch, base: lease.workspace.base.commit, from: 'a'.repeat(40), to: 'b'.repeat(40), commits: [], ...fields }; +} + /** The assertions a runner lends the interpreter, so the scenarios do not depend on a test framework. */ export type ConformanceExpect = (actual: unknown, message?: string) => { toMatchObject(expected: Record): void; @@ -117,7 +131,7 @@ export async function runConformanceScenario(scenario: ConformanceScenario, runn const started = await client.start(scenario.workflowName); if ('error' in started) throw new Error(started.error.message); const runId = String(started.data.run); - const leases = new Map(); + const leases = new Map(); const attemptOf = (name: string): string | undefined => leases.get(name)?.attempt; for (const operation of scenario.operations) { switch (operation.kind) { @@ -140,7 +154,7 @@ export async function runConformanceScenario(scenario: ConformanceScenario, runn if (operation.cancelled) expect('data' in envelope && envelope.data.cancelled).toEqual(operation.cancelled.map(attemptOf)); for (const field of operation.absent ?? []) expect('data' in envelope ? envelope.data.lease : undefined).not.toHaveProperty(field); if ('data' in envelope && envelope.data.lease) { - leases.set(operation.saveAs, envelope.data.lease as { attempt: string; revision: number }); + leases.set(operation.saveAs, SavedLeaseSchema.parse(envelope.data.lease)); } break; } @@ -165,6 +179,7 @@ export async function runConformanceScenario(scenario: ConformanceScenario, runn revision: operation.revision ?? lease?.revision, driver: profileDriver(operation.as ?? 'local'), ...operation.body, + ...(operation.changes ? { changes: leasedChanges(lease, operation.changes) } : {}), })).toMatchObject(operation.expect); break; } diff --git a/src/workflow/conformance.ts b/src/workflow/conformance.ts index ae85e1d13..848241866 100644 --- a/src/workflow/conformance.ts +++ b/src/workflow/conformance.ts @@ -14,6 +14,25 @@ steps: ok: boolean `; +const REVIEW_LINK = `schemaVersion: v1alpha1 +name: review-link +description: A remote driver reports a review link. +steps: + - id: work + workspace: new + command: echo work +`; + +const REPORTED_PR = `schemaVersion: v1alpha1 +name: reported-pr +description: A remote driver opens the pull request of the branch it worked on and reports it. +steps: + - id: work + workspace: new + command: echo work + deliver: pr +`; + const FAILED_DEPENDENCY = `schemaVersion: v1alpha1 name: build description: A step whose dependency failed is skipped. @@ -187,6 +206,23 @@ steps: /** Every protocol scenario a driver must pass, the ones each concern keeps in its own file included. */ export const workflowConformanceScenarios: ConformanceScenario[] = [ + { + name: 'a pull request a driver reports is kept, without its query or fragment, for the step to deliver', + workflowName: 'reported-pr', yaml: REPORTED_PR, + operations: [ + { kind: 'next', saveAs: 'work', base: { ref: 'main', commit: 'a'.repeat(40) }, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', changes: { pr_url: 'https://review.example/changes/1?session=private#note', unknown: 'private' }, body: { outcome: 'success' }, expect: { data: { accepted: true, step: { id: 'work', status: 'delivering', outcome: 'success' }, ignored: ['changes.unknown'] } } }, + ], + expectState: { steps: [{ id: 'work', status: 'delivering', delivery: { reported: 'https://review.example/changes/1' } }] }, + }, + { + name: 'an unsafe review URL is dropped without rejecting the report', workflowName: 'review-link', yaml: REVIEW_LINK, + operations: [ + { kind: 'next', saveAs: 'work', base: { ref: 'main', commit: 'a'.repeat(40) }, expect: { data: { lease: { step: 'work' } } } }, + { kind: 'report', lease: 'work', changes: { pr_url: 'https://user:token@review.example/changes/1' }, body: { outcome: 'success' }, expect: { data: { accepted: true, dropped: [{ field: 'changes.pr_url', reason: 'contains credentials' }] } } }, + { kind: 'next', saveAs: 'done', expect: { data: { done: { outcome: 'succeeded' } } } }, + ], + }, { name: 'two-step command success', workflowName: 'build', @@ -831,6 +867,99 @@ export const workflowGitConformanceScenarios: GitConformanceScenario[] = [ }, ]; +/** What a run holds once a delivery case ends, over every install that drove it. */ +export interface DeliveryExpectation { + /** Pull requests TeamAI asked the host to open. */ + creates: number; + run: 'succeeded' | 'failed' | 'cancelled'; + work: 'succeeded' | 'failed' | 'cancelled'; + /** The pull requests the run lists in `deliveries` and its end record's `prs`. */ + prs: number; + /** The failure code of `work`, whose message then names its branch. */ + failure?: string; +} + +/** + * A delivery case for every driver that takes workspace leases, run against `deliveryConformanceWorkflow` with a fake + * provider every case shares. The runner gives the product an `https://github.com//` origin that git + * rewrites to a bare repository, and a fake `gh` that records each create and answers lookups from what it recorded. + * With `hold`, the fake waits inside that call, a create once it recorded the request, until the runner has done + * `action`: `crash` kills the process the call came from, and the driver calls `next` again; `publish` publishes the + * run and kills that process, and a driver of a second install with its own checkout claims the run and drives it; + * `cancel` cancels the run; `poll` holds a delivery that another process of the machine started after `work` reported, + * while the driver under test calls `next`. A driver whose delivery never reaches `hold` runs to the end untouched. + * `unreachable` fails every lookup. `opened` is expected of a driver that leaves the pull request to TeamAI, `reported` + * of one that reports its own as `changes.pr_url`. After every case, `work`'s branch is on origin at its reported commit. + */ +export interface DeliveryConformanceScenario { + name: string; + hold?: { call: 'lookup' | 'create'; action: 'crash' | 'publish' | 'cancel' | 'poll' }; + unreachable?: true; + opened: DeliveryExpectation; + reported: DeliveryExpectation; +} + +/** `ship` copies what the host holds when it runs to `SHIP_SAW`, which the runner sets: the open requests by then. */ +export const deliveryConformanceWorkflow = `schemaVersion: v1alpha1 +name: deliver +description: A step delivers its branch as a pull request, and the step that needs it runs once it is delivered. +steps: + - id: work + workspace: new + command: node -e "require('fs').writeFileSync('work.txt', 'W')" + deliver: pr + - id: ship + needs: [work] + command: node -e "require('fs').copyFileSync(process.env.SHIP_OPEN, process.env.SHIP_SAW)" +`; + +const DELIVERED: DeliveryExpectation = { creates: 1, run: 'succeeded', work: 'succeeded', prs: 1 }; +const REPORTED: DeliveryExpectation = { creates: 0, run: 'succeeded', work: 'succeeded', prs: 1 }; + +export const workflowDeliveryConformanceScenarios: DeliveryConformanceScenario[] = [ + { + name: 'a delivered branch opens one pull request into the run base, and the step that needs it runs once it is delivered', + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a delivery whose process stops inside the create finds the pull request on the next call and opens no other', + hold: { call: 'create', action: 'crash' }, + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a run published while it delivers, and claimed by another install with its own checkout, opens one pull request', + hold: { call: 'create', action: 'publish' }, + opened: DELIVERED, + reported: REPORTED, + }, + { + name: 'a run cancelled while its delivery looks for an open pull request opens none', + hold: { call: 'lookup', action: 'cancel' }, + opened: { creates: 0, run: 'cancelled', work: 'cancelled', prs: 0 }, + reported: { creates: 0, run: 'cancelled', work: 'cancelled', prs: 0 }, + }, + { + name: 'a run cancelled while its delivery opens the pull request lists it, and the step stays cancelled', + hold: { call: 'create', action: 'cancel' }, + opened: { creates: 1, run: 'cancelled', work: 'cancelled', prs: 1 }, + reported: REPORTED, + }, + { + name: 'a delivery that cannot reach the host fails naming the branch, which stays on origin', + unreachable: true, + opened: { creates: 0, run: 'failed', work: 'failed', prs: 0, failure: 'delivery_failed' }, + reported: { creates: 0, run: 'failed', work: 'failed', prs: 0, failure: 'delivery_failed' }, + }, + { + name: 'a driver polls while another process of its machine delivers, and gets the step that needs it once delivered', + hold: { call: 'create', action: 'poll' }, + opened: DELIVERED, + reported: REPORTED, + }, +]; + export const judgeLeaseConformanceScenario = { name: 'a judge lease renews through running and settles only through score submission', yaml: `schemaVersion: v1alpha1 diff --git a/src/workflow/delivery-ledger.ts b/src/workflow/delivery-ledger.ts new file mode 100644 index 000000000..ded06261c --- /dev/null +++ b/src/workflow/delivery-ledger.ts @@ -0,0 +1,125 @@ +/** The run's delivery receipts and unresolved requests, independent of locks and network calls. */ +import type { PrUrl } from './pr-url.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; +import { deliveryFailure, type DeliveryJob, type DeliveryOutcome, type findDelivered } from './pr-delivery.js'; +import { delivered, deliveryFailed, type DeliveringStep } from './step-state.js'; +import type { RunState } from './types.js'; +import { branchName, childWorkOff, deliveryTarget } from './workspace.js'; + +/** The delivery of `step`, as the drain and the outcome it applies name it. */ +function deliveryKey(step: DeliveringStep): DeliveryKey { + return { step: step.id, pass: step.pass ?? 1, settledAt: step.settledAt, branch: step.delivery.branch, head: step.delivery.head }; +} + +/** Whether `a` and `b` name one delivery: the same pass of the same step, settled at the same revision, of one commit. */ +function sameKey(a: DeliveryKey, b: DeliveryKey): boolean { + return a.step === b.step && a.pass === b.pass && a.settledAt === b.settledAt && a.branch === b.branch && a.head === b.head; +} + +/** The step `key` names, while it still delivers that pass of that commit. */ +export function deliveringStep(run: RunState, key: DeliveryKey): DeliveringStep | undefined { + return run.steps.find((item): item is DeliveringStep => item.status === 'delivering' && sameKey(deliveryKey(item), key)); +} + +/** + * The first delivery to run, after what reports and controls left pending is applied, so a step an end or a cancel + * stopped is not delivered. One in `waiting`, whose branch this drain found ahead of the recorded commit, waits for a + * later drain, and so does one whose branch a leased step writes, since that step's next push would enter its pull + * request unchecked. Undefined when nothing delivers, or when the run settles, which the locked body of `next` finishes. + */ +export function nextDelivery( + run: RunState, reconcile: (run: RunState) => object, writerLeased: (run: RunState, branch: string) => boolean, waiting: readonly DeliveryKey[], +): DeliveryJob | undefined { + if (run.status !== 'running') return undefined; + const settledRun = reconcile(run); + if ('unreadable' in settledRun || 'outcome' in settledRun) return undefined; + for (const step of run.steps) { + if (step.status !== 'delivering') continue; + const key = deliveryKey(step); + if (waiting.some((ahead) => sameKey(ahead, key)) || writerLeased(run, key.branch)) continue; + const target = deliveryTarget(run, key.branch); + if (!run.checkout || !run.base || target === undefined) { + deliveryFailed(step, deliveryFailure({ runId: run.id, key }, { kind: 'no_checkout' })); + continue; + } + return { + key, runId: run.id, runName: run.name, checkout: run.checkout, base: run.base.commit, target, + recordedHead: run.copies?.[key.branch]?.commit, childWork: childWorkOff(run, step, key.branch), + openedBy: run.steps.find((item) => item !== step && branchName(run.id, item.id, item.pass ?? 1) === key.branch)?.id, + ...(step.delivery.reported ? { reported: step.delivery.reported } : {}), + }; + } + return undefined; +} + +/** The first request in `run.creating` that this drain has not looked up yet. */ +export function unresolvedCreate(run: RunState, tried: readonly Omit[]): { create: UnresolvedCreate; checkout: string } | undefined { + const { checkout } = run; + const create = run.creating?.find((listed) => !tried.some((done) => sameCreate(done, listed))); + return create && checkout ? { create, checkout } : undefined; +} + +export function sameCreate(a: Omit, b: Omit): boolean { + return sameKey(a, b) && a.target === b.target; +} + +/** Drops `create` from the requests the run has no receipt for. */ +function forgetCreate(run: RunState, create: Omit): void { + const left = run.creating?.filter((listed) => !sameCreate(listed, create)); + if (left?.length) run.creating = left; + else delete run.creating; +} + +/** Lists `url` as what `key` delivered into `target`, once; true when the run had no receipt for it yet. */ +function recordReceipt(run: RunState, key: DeliveryKey, target: string, url: PrUrl): boolean { + if (run.deliveries?.some((receipt) => receipt.url === url)) return false; + (run.deliveries ??= []).push({ url, branch: key.branch, base: target, step: key.step, pass: key.pass }); + return true; +} + +/** + * Applies a delivery's outcome to the job it was picked as. A pull request gets its receipt whatever became of the step + * meanwhile: one opened while a cancel landed cannot be taken back, so the run says it exists. The receipt settles the + * job's listed request; a failed delivery leaves it listed, since a create that failed may still have opened one. The + * step settles only while it still delivers what `job` delivered, at the revision its agent's report settled it. Origin's + * branch ahead of the recorded commit fails it only once no report of that tip can come: no step that writes the branch + * is leased, and no report recorded another commit on it since the job was picked. Until then a pull request settles + * nothing either, and one the job did not open gets no receipt: a later call checks the branch again and finds it. + * `requeueEnd` is set when the receipt reached a run that already ended, whose end record must go out again to list it. + */ +export function completeDelivery( + run: RunState, job: DeliveryJob, outcome: DeliveryOutcome, + reconcile: (run: RunState) => unknown, writerLeased: (run: RunState, branch: string) => boolean, +): { requeueEnd: boolean } { + const create = { ...job.key, target: job.target }; + const unchecked = (): boolean => writerLeased(run, job.key.branch) || run.copies?.[job.key.branch]?.commit !== job.recordedHead; + const settles = !unchecked(); + // A request this job opened gets its receipt whatever became of the branch; one it found or was told of waits. + const taken = settles || run.creating?.some((listed) => sameCreate(listed, create)) === true; + const receipted = outcome.ok && taken && recordReceipt(run, job.key, job.target, outcome.pr); + if (outcome.ok && taken) forgetCreate(run, create); + if (run.status === 'running') reconcile(run); + const step = deliveringStep(run, job.key); + const awaited = !outcome.ok && outcome.fault.kind === 'branch_ahead' && unchecked(); + if (step && outcome.ok && settles) delivered(step); + else if (step && !outcome.ok && !awaited) deliveryFailed(step, deliveryFailure(job, outcome.fault)); + return { requeueEnd: receipted && run.status !== 'running' }; +} + +/** + * Applies what a lookup said of a request in `run.creating`, and touches no step. One the host holds gets its receipt, + * and unsafe URLs are dropped. Empty and failed answers stay until the second empty or third failed lookup. + */ +export function resolveCreate(run: RunState, create: UnresolvedCreate, found: Awaited>): { requeueEnd: boolean } { + const listed = run.creating?.find((item) => sameCreate(item, create)); + if (!listed) return { requeueEnd: false }; + const lookups = listed.lookups ?? { total: 0, none: 0, failed: 0 }; + listed.lookups = { + total: lookups.total + 1, + none: lookups.none + (found.kind === 'none' ? 1 : 0), + failed: lookups.failed + (found.kind === 'failed' ? 1 : 0), + }; + const receipted = found.kind === 'found' && recordReceipt(run, create, create.target, found.pr); + if (found.kind === 'found' || found.kind === 'unsafe' || listed.lookups.none >= 2 || listed.lookups.failed >= 3) forgetCreate(run, create); + return { requeueEnd: receipted && run.status !== 'running' }; +} diff --git a/src/workflow/delivery-schema.ts b/src/workflow/delivery-schema.ts new file mode 100644 index 000000000..0adc84a15 --- /dev/null +++ b/src/workflow/delivery-schema.ts @@ -0,0 +1,29 @@ +/** What a run file and a snapshot keep of delivery, as schemas only, so neither loads delivery code or a provider. */ +import { z } from 'zod'; +import { PrUrlSchema } from './pr-url.js'; + +/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ +export const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); + +/** A pull request delivered on a run: its URL, the branch it was opened from and into, and the step pass it delivered. */ +export const DeliveryReceiptSchema = z.object({ + url: PrUrlSchema, + branch: z.string(), + base: z.string(), + step: z.string(), + pass: z.number().int(), +}); +export type DeliveryReceipt = z.infer; + +/** Which delivery an outcome belongs to while the step still delivers this pass of this commit. */ +const DeliveryKeySchema = z.object({ + step: z.string(), pass: z.number().int(), settledAt: z.number().int(), branch: z.string().min(1), head: CommitIdSchema, +}); +export type DeliveryKey = z.infer; + +/** A create the host may have accepted, retained independently of the step's lifecycle. */ +export const UnresolvedCreateSchema = DeliveryKeySchema.extend({ + target: z.string().min(1), + lookups: z.object({ total: z.number().int().nonnegative(), none: z.number().int().nonnegative(), failed: z.number().int().nonnegative() }).default({ total: 0, none: 0, failed: 0 }), +}); +export type UnresolvedCreate = z.infer; diff --git a/src/workflow/engine.ts b/src/workflow/engine.ts index 325f26d2a..431e051b9 100644 --- a/src/workflow/engine.ts +++ b/src/workflow/engine.ts @@ -18,21 +18,28 @@ import { } from './template.js'; import { parseStepPath } from './step-path.js'; import { - ReadSchema, answerHitl, fail, lease, release, settled, skip, start, stopAttempt, stopWhere, stopped, succeed, waitForAnswer, - type ItemResult, type LeasedStep, type StepRead, type StepState, + ReadSchema, answerHitl, fail, inProgress, lease, release, settled, skip, start, startDelivery, stopAttempt, stopWhere, stopped, + succeed, waitForAnswer, type ItemResult, type LeasedStep, type StepRead, type StepState, } from './step-state.js'; import { budgetSpent, countLateTokens, expire, itemsSpent, leaseDeadline, ownDeadline, startLimits } from './limits.js'; import { addItems, afterStanding, expandItems, isItem, itemsAfter, itemsOf, itemsSettled, startedItems } from './fan-out.js'; import { decideRound, decideStepLoop, exhaustedAtStart, firstPass, isUnder, loopSettled } from './loop.js'; import { childScope, contextOf, insertChildren, localId, statesIn, stepDefinitions, topScope, type Scope } from './scope.js'; -import { planWorkspace, plannedBranch, sessionOf, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspacePlan } from './workspace.js'; +import { deliverableOf, deliveryProblem, deliveryTarget, planWorkspace, plannedBranch, sessionOf, workLeftUndelivered, type LeaseSession, type SessionRef, type WorkspaceContract, type WorkspacePlan } from './workspace.js'; +import { + deliverPr, deliveryFailure, findDelivered, reportedPrMismatch, type DeliveryJob, +} from './pr-delivery.js'; +import { sanitizePrUrl, type PrUrl } from './pr-url.js'; +import { completeDelivery, deliveringStep, nextDelivery, resolveCreate, sameCreate, unresolvedCreate } from './delivery-ledger.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; import { CAN_KEYS, CanSchema, ReportedUnenforcedSchema, heldFor, mergeRequirements, negotiate, reportedUnenforced, requiredEnvKeys, unionUnenforced, type Allow, type DriverCan, type EnvFacts, type Held, type Negotiation, type ReportedUnenforced, type Terms, } from './requirements.js'; import { knownSecrets, resolveWorkflowEnv, secretKey, type WorkflowEnv } from '../env-resolution.js'; import { redactJson } from '../utils/redact.js'; -import { generateRunId, installOwner, isRunId, updateRun } from './store.js'; +import { deliveryLockPath, generateRunId, installOwner, isRunId, readRun, updateRun } from './store.js'; +import { withFileLock } from '../update.js'; import { pinScorers } from '../scorers/cache.js'; import { isLeaseRef } from '../scorers/entry.js'; import type { @@ -72,6 +79,8 @@ export interface ReportInput { failure?: WorkflowFailure; /** The branch this attempt committed, when it ran in a workspace. */ changes?: { branch: string; base: string; from: string; to: string; commits: string[] }; + /** `changes.pr_url` as the driver sent it: a pull request it opened itself, kept only once it is sanitized. */ + prUrl?: unknown; /** The branch and commit this attempt read, when it ran read-only in a workspace. */ read?: StepRead; agent?: ReportedAgent; @@ -281,13 +290,20 @@ function leaseData( } }; } -/** Records who opened a workspace the first time a driver takes its branch, or who first leases a copy a claim brought in. */ +/** + * Records who opened a workspace the first time a driver takes its branch, or who first leases a copy a claim brought in, + * and, before the copy has a commit, the branch it starts from. + */ function rememberCopy(run: RunState, plan: WorkspacePlan, driver: DriverIdentity): void { const branch = plan.contract?.branch; if (!branch) return; + const from = plan.contract?.start.branch ?? plan.contract?.merge[0]?.branch; const copy = (run.copies ??= {})[branch]; - if (copy) copy.owner ??= driver; - else run.copies[branch] = { owner: driver }; + if (copy) { + copy.owner ??= driver; + if (!copy.from && !copy.commit && from && from !== branch) copy.from = from; + } + else run.copies[branch] = { owner: driver, ...(from && from !== branch ? { from } : {}) }; } /** Whether a step holds a lease in copy `branch`. */ @@ -535,17 +551,30 @@ function startWorkflowStep( /** * A step takes how what it ran settled: a workflow step, the outcome, the end, and the outputs of its workflow; a - * `for_each` step, the outcome and the results of its items. + * `for_each` step, the outcome and the results of its items. A workflow step or `for_each` item that delivers goes on + * to deliver what it wrote; a `for_each` step delivers nothing of its own, its items do. */ -function finishStep(state: StepState, result: WorkflowResult): void { +function finishStep(run: RunState, state: StepState, definition: WorkflowStep, result: WorkflowResult): void { const settlement = { settledAt: result.settledAt, ...(result.end !== undefined ? { end: result.end } : {}), ...(result.outputs !== undefined ? { outputs: result.outputs } : {}), ...(result.results ? { results: result.results } : {}), }; - if (result.outcome === 'succeeded') succeed(state, settlement); - else fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); + if (result.outcome !== 'succeeded') { + fail(state, { ...settlement, ...(result.failure ? { message: result.failure.message, failure: result.failure } : {}) }); + return; + } + const delivers = definition.deliver === 'pr' && (!definition.forEach || isItem(state)); + const delivery = delivers ? deliverableOf(run, state, definition) : undefined; + // A branch that holds nothing beyond the run base opens no pull request, so work a step inside wrote elsewhere would be lost. + const left = delivers && !delivery && definition.workflow !== undefined ? workLeftUndelivered(run, state) : undefined; + if (delivery) startDelivery(state, settlement, delivery); + else if (left) { + const key = { step: state.id, pass: state.pass ?? 1, settledAt: result.settledAt, branch: left.branch, head: left.head }; + const failure = deliveryFailure({ runId: run.id, key }, { kind: 'child_work_left_out', step: left.child.step, branch: left.child.branch }); + fail(state, { ...settlement, message: failure.message, failure }); + } else succeed(state, settlement); } function childOrUnreadable(run: RunState, scope: Scope, states: StepState[], state: StepState, definition: WorkflowStep): Scope | { unreadable: string } { @@ -570,7 +599,7 @@ function advanceInstance( if (state.status === 'running') { const result = settleRunning(run, scope, states, state, definition, now); if (!('outcome' in result)) return result; - finishStep(state, result); + finishStep(run, state, definition, result); return undefined; } if (definition.forEach && !isItem(state)) { @@ -759,7 +788,7 @@ function settleScope(run: RunState, scope: Scope, now: string): Settled { return decided === 'ended' ? result : { ...result, outcome: 'failed', failure: decided }; } if (ready.length > 0) return { ready }; - if (states.some((item) => item.status === 'waiting' || item.status === 'leased' || item.status === 'running')) return { inProgress: true }; + if (states.some(inProgress)) return { inProgress: true }; return { unreadable: `Run ${run.id} has pending steps whose needs cannot be met` }; } } @@ -837,6 +866,8 @@ export async function startRun( if (mismatch) return error('invalid_inputs', mismatch); const tree = await loadWorkflowTree(config, cached); if ('failed' in tree) return error(tree.failed.code, tree.failed.message); + const undeliverable = deliveryProblem(cached.definition, tree.workflows, options.base); + if (undeliverable) return error(undeliverable.code, undeliverable.message); const scorers = await pinScorers(config, listedScorers([cached.definition, ...Object.values(tree.workflows)])); const pinned = Object.fromEntries([ ...[cached.definition, ...Object.values(tree.workflows)].map((definition) => [`workflows/${definition.name}.yaml`, definition]), @@ -937,16 +968,99 @@ export interface NextOptions { env?: WorkflowEnv; } +/** + * The gate before a pull request is opened: with what reports and controls left pending applied, whether the step still + * delivers `job`, the branch's recorded commit is still the one `job` checked, and no step that writes the branch is + * leased. If so, the run lists its request in the same write, so one opened by a process that then stops is looked up, + * whatever becomes of the step, rather than lost. + */ +async function markCreating(config: LocalConfig, runId: string, secrets: Record, job: DeliveryJob): Promise<'open' | 'refused' | 'busy'> { + const marked = await updateRun(config, runId, secrets, async (run): Promise<{ value: 'open' | 'refused' | 'busy'; write?: RunState }> => { + if (run === 'missing' || run.status !== 'running') return { value: 'refused' }; + const now = new Date().toISOString(); + const settledRun = reconcileRun(run, now); + const step = 'unreadable' in settledRun || 'outcome' in settledRun ? undefined : deliveringStep(run, job.key); + if (!step) return { value: 'refused' }; + if (run.copies?.[job.key.branch]?.commit !== job.recordedHead || branchWriterLeased(run, job.key.branch)) return { value: 'busy' }; + const create = { ...job.key, target: job.target }; + const listed = run.creating?.find((item) => sameCreate(item, create)); + // A request listed before is asked again now, so its lookups count from this create. + const lookups = { total: 0, none: 0, failed: 0 }; + if (listed) listed.lookups = lookups; + else (run.creating ??= []).push({ ...create, lookups }); + return { value: 'open', write: { ...run, updatedAt: now } }; + }); + return marked.status === 'done' ? marked.value : 'busy'; +} + +/** + * Delivers the run's delivering steps one at a time, before `next` answers, whichever driver called it, then looks up + * each request in `run.creating` once, on an ended run too. The run lock is held to pick a delivery and to apply its + * outcome, never across git or a provider. While another process of this machine delivers, it holds the delivery lock, + * and this call leaves the work to it. + */ +async function drainDeliveries(config: LocalConfig, runId: string, secrets: Record): Promise { + if (!isRunId(runId)) return; + const loaded = await readRun(config, runId); + if (typeof loaded === 'string') return; + const delivering = loaded.status === 'running' && loaded.steps.some((step) => step.status === 'delivering'); + if (!delivering && !unresolvedCreate(loaded, [])) return; + const drained = await withFileLock(deliveryLockPath(config, runId), { attempts: 1, delayMs: 0 }, async () => { + let requeued = false; + const tried: Array> = []; + const waiting: DeliveryKey[] = []; + for (;;) { + const picked = await updateRun(config, runId, secrets, async (run) => { + if (run === 'missing') return { value: undefined }; + const before = JSON.stringify(run); + const now = new Date().toISOString(); + const job = nextDelivery(run, (item) => reconcileRun(item, now), branchWriterLeased, waiting); + const value = job ? { job } : unresolvedCreate(run, tried); + return { value, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: now } } : {}) }; + }); + if (picked.status !== 'done' || !picked.value) return requeued; + let apply: (run: RunState, now: string) => { requeueEnd: boolean }; + if ('job' in picked.value) { + const { job } = picked.value; + tried.push({ ...job.key, target: job.target }); + const outcome = await deliverPr(job, () => markCreating(config, runId, secrets, job)); + // As when the pick is busy: the step still delivers, and a later call checks and delivers it. + if (outcome === 'busy') return requeued; + if (!outcome.ok && outcome.fault.kind === 'branch_ahead') waiting.push(job.key); + apply = (run, now) => completeDelivery(run, job, outcome, (item) => reconcileRun(item, now), branchWriterLeased); + } else { + const { create, checkout } = picked.value; + tried.push(create); + const found = await findDelivered(checkout, create); + apply = (run) => resolveCreate(run, create, found); + } + const applied = await updateRun(config, runId, secrets, async (run) => { + if (run === 'missing') return { value: undefined }; + const before = JSON.stringify(run); + const now = new Date().toISOString(); + const { requeueEnd } = apply(run, now); + // Queued under the lock with the receipt, as finishRun queues an end; it publishes once the lock is released. + if (requeueEnd) run.endRecordQueued = await queueRunRecord(config, run, run.driver, 'end', secrets); + return { value: requeueEnd, ...(JSON.stringify(run) !== before ? { write: { ...run, updatedAt: now } } : {}) }; + }); + if (applied.status !== 'done' || applied.value === undefined) return requeued; + requeued ||= applied.value; + } + }); + if (drained.held && drained.value) await publishRunRecords(config); +} + export async function leaseNext(config: LocalConfig, runId: string, driver: DriverIdentity, options: NextOptions = {}): Promise { driver = { kind: driver.kind, machine: driver.machine, workspace: driver.workspace }; if (options.claim) { const claimed = await claimRun(config, runId, { checkout: driver.workspace }); if (!claimed.ok) return { error: claimed.error }; } + // Resolved once, so the env this `next` offers against and the secrets its writes, deliveries' included, mask agree. + const env = options.env ?? await resolveWorkflowEnv(config); + await drainDeliveries(config, runId, env.secrets); const running = new Set(options.running); const can = options.can ?? {}; - // Resolved once, so the env this `next` offers against and the secrets its writes mask agree. - const env = options.env ?? await resolveWorkflowEnv(config); const context: OfferContext = { can, allow: config.workflows?.allow ?? {}, facts: env.facts }; const maxParallel = config.workflows?.max_parallel; // The end record is queued under the run lock, so the run stores whether it @@ -971,9 +1085,14 @@ export async function leaseNext(config: LocalConfig, runId: string, driver: Driv queued = await finishRun(config, run, now, result, env.secrets, driver); return answer(doneData(run)); }; - const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => ( - answer({ wait: { reason: open ? `${open.id} is leased` : 'canTakeLease is false' } }) - ); + const wait = (open = run.steps.find((step) => step.status === 'leased')): { save: boolean; envelope: WorkflowEnvelope } => { + // Steps still delivering here are being delivered by another process of this machine, waited for a busy run + // lock, or wait for a report: of the commit their branch is ahead at on origin, or of a leased step that writes + // their branch. A driver polls until they settle. + const delivering = run.steps.filter((step) => step.status === 'delivering').map((step) => step.id); + const reason = open ? `${open.id} is leased` : delivering.length > 0 ? `${delivering[0]} is delivering` : 'canTakeLease is false'; + return answer({ wait: { reason, ...(delivering.length > 0 ? { delivering } : {}) } }); + }; if (run.status !== 'running') { // Settled by an earlier next, which queued the end record unless usage // reporting was off; the done envelope reports that record's state. @@ -1211,6 +1330,7 @@ export function parseReport(value: unknown): ParsedReport | string { return 'changes.commits must be an array of strings'; } report.changes = { branch: value.branch, base: value.base, from: value.from, to: value.to, commits: value.commits }; + if (value.pr_url !== undefined) report.prUrl = value.pr_url; } if (record.read !== undefined) { const read = ReadSchema.safeParse(record.read); @@ -1232,7 +1352,12 @@ export function parseReport(value: unknown): ParsedReport | string { if (!unenforced.success) return 'unenforced must list the requirements the driver did not enforce: max_turns'; report.unenforced = unenforced.data; } - return { report, ignored: unreadFields(record, REPORT_FIELDS) }; + const ignored = unreadFields(record, REPORT_FIELDS); + if (isRecord(record.changes)) { + const known = new Set(['branch', 'base', 'from', 'to', 'commits', 'pr_url']); + ignored.push(...Object.keys(record.changes).filter((key) => !known.has(key)).map((key) => `changes.${key}`)); + } + return { report, ignored }; } /** A `next` payload: who the driver is, and what else it tells `next`. */ @@ -1322,13 +1447,25 @@ function branchStillUsed(run: RunState, branch: string, now: string): boolean { const ahead = structuredClone(run); if ('unreadable' in reconcileRun(ahead, now)) return true; for (const item of ahead.steps) { - if (item.status === 'running' || settled(item)) continue; + if (item.status === 'running' || item.status === 'delivering' || settled(item)) continue; const located = locateStep(ahead, item.id); if (located && plannedBranch(ahead, located.scope, item, located.step) === branch) return true; } return false; } +/** Whether a leased step writes `branch`, so a commit it pushed before it reported may already be on origin. */ +function branchWriterLeased(run: RunState, branch: string): boolean { + return run.steps.some((item) => { + if (item.status !== 'leased') return false; + const located = locateStep(run, item.id); + const plan = located && planWorkspace(run, located.scope, item, located.step, { + driver: item.lease.driver, modes: ['read-only', 'new', 'continue'], + }, true); + return plan !== undefined && plan.mode !== 'read-only' && plan.contract?.branch === branch; + }); +} + /** * The session chains the steps that have not settled will continue, once each: each chain is the session a step resumes * with the sessions it continued before, newest first. Like `branchStillUsed`, it reads the run as the next `next` settles @@ -1382,6 +1519,20 @@ function continueSession(run: RunState, report: ReportInput): void { (run.sessionContinuations ??= []).push({ from, to }); } +/** + * Why the URL a delivering step's report keeps after sanitizing is still dropped, read from the run before its lock is + * taken, since the provider's lookup is a network call. The locked report checks the step again. + */ +async function reportedPrUnmatched(config: LocalConfig, runId: string, report: ReportInput, url: PrUrl): Promise { + if (report.outcome !== 'success' || !report.changes || !isRunId(runId)) return undefined; + const run = await readRun(config, runId); + if (typeof run === 'string' || !run.checkout) return undefined; + const target = deliveryTarget(run, report.changes.branch); + const step = run.steps.find((item) => item.status === 'leased' && item.lease.attempt === report.attempt); + if (!step || target === undefined || locateStep(run, step.id)?.step.deliver !== 'pr') return undefined; + return reportedPrMismatch({ checkout: run.checkout, url, branch: report.changes.branch, target }); +} + /** * Counts the tokens a report of an attempt the engine stopped says it spent, so a budget does not undercount the work a * stop cut short. Only the driver that held the lease counts, and only once. Returns whether it counted them. @@ -1399,6 +1550,8 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: return error('judge_lease', `Attempt ${report.attempt} is a judge lease, which workflow report does not settle. Submit its answer with: teamai score --lease ${report.attempt} --submit --stdin`); } const install = await installOwner(config); + const reported = report.prUrl === undefined ? undefined : sanitizePrUrl(report.prUrl); + const unmatched = reported?.url ? await reportedPrUnmatched(config, runId, report, reported.url) : undefined; return withRun(config, runId, secrets ?? await knownSecrets(config), async (run, now) => { // Settled membership wins. An old revision of a settled attempt is still repeated_report. if (run.settledAttempts.includes(report.attempt)) { @@ -1491,11 +1644,22 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: if (contract && (report.outcome === 'success' || report.exitCode !== undefined || report.changes || report.read)) { ((run.copies ??= {})[contract.branch] ??= { owner: report.driver }).used = true; } + const successful = { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }; + const delivery = outcome === 'success' && defined?.deliver === 'pr' ? deliverableOf(run, step, defined, report.changes) : undefined; const settledStep = outcome === 'released' ? release(step) - : outcome === 'success' ? succeed(step, { ...settlement, ...(report.outputs !== undefined ? { outputs: report.outputs } : {}) }) - : fail(step, { ...settlement, failure }); - // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. + : outcome !== 'success' ? fail(step, { ...settlement, failure }) + : delivery ? startDelivery(step, successful, { ...delivery, ...(reported?.url && !unmatched ? { reported: reported.url } : {}) }) + : succeed(step, successful); + // From the lease, so a reader that was the copy's last user lets the driver remove the worktree. Delivery reads the + // pushed branch, so it does not keep the worktree. const cleanup = outcome === 'success' && contract !== undefined && !branchStillUsed(run, contract.branch, now); + // A URL that is not kept, or that no delivery uses, is named without its value, which may hold a token, and the + // report still counts. + const dropReason = reported?.reason ?? unmatched ?? (reported?.url === undefined || delivery ? undefined + : defined?.deliver !== 'pr' ? 'the step has no deliver: pr' + : outcome === 'released' ? 'the step was released' + : outcome !== 'success' ? 'the step failed' + : 'the branch has no changes beyond the run base'); return { save: true, envelope: ok({ @@ -1503,6 +1667,7 @@ export async function reportAttempt(config: LocalConfig, runId: string, report: revision: run.revision, step: { id: settledStep.id, status: settledStep.status, outcome, ...(settlement.unenforced ? { unenforced: settlement.unenforced } : {}) }, ...(cleanup ? { cleanup: true } : {}), + ...(dropReason ? { dropped: [{ field: 'changes.pr_url', reason: dropReason }] } : {}), }), }; }); diff --git a/src/workflow/fan-out.ts b/src/workflow/fan-out.ts index 459339c29..4c31ccec5 100644 --- a/src/workflow/fan-out.ts +++ b/src/workflow/fan-out.ts @@ -93,7 +93,7 @@ export function afterStanding(after: StepState[]): 'ready' | 'waiting' | 'skip' return after.every((item) => item.status === 'succeeded') ? 'ready' : 'waiting'; } -/** How many items are started and not yet settled, which `max_parallel` bounds. */ +/** How many items have an agent or workflow at work, which `max_parallel` bounds. An item that delivers has none. */ export function startedItems(items: StepState[]): number { return items.filter((item) => item.status === 'leased' || item.status === 'running').length; } @@ -127,6 +127,7 @@ export function itemsSettled( function branchesFromChildren(run: RunState): Map { const best = new Map(); for (const step of run.steps) { + if (step.status === 'cancelled') continue; const branch = 'changes' in step ? step.changes?.branch : undefined; if (!branch) continue; const settledAt = 'settledAt' in step ? step.settledAt : 0; diff --git a/src/workflow/local-driver.ts b/src/workflow/local-driver.ts index e76c62389..5cd155880 100644 --- a/src/workflow/local-driver.ts +++ b/src/workflow/local-driver.ts @@ -1150,12 +1150,15 @@ async function driveRun(config: LocalConfig, runId: string, cwd: string, options } if ('lease' in next.data && room()) continue; if (inFlight.size === 0 && dying.size === 0 && !prompt && !question) { + const wait = isRecord(next.data.wait) ? next.data.wait : {}; + // Another process of this machine delivers a step this run waits for; it settles without this driver. + if (Array.isArray(wait.delivering) && wait.delivering.length > 0) { + await new Promise((resolve) => setTimeout(resolve, pollMs)); + continue; + } // A wait for a call that could not take a lease says nothing about what is left; ask again with room. if (!canTakeLease) continue; - const reason = typeof (next.data.wait as { reason?: unknown }).reason === 'string' - ? (next.data.wait as { reason: string }).reason - : 'the next step is not available'; - return { error: { code: 'busy', message: reason } }; + return { error: { code: 'busy', message: typeof wait.reason === 'string' ? wait.reason : 'the next step is not available' } }; } let timer: NodeJS.Timeout | undefined; const exited = await Promise.race([ diff --git a/src/workflow/loop.ts b/src/workflow/loop.ts index df00e5fab..0d23314fd 100644 --- a/src/workflow/loop.ts +++ b/src/workflow/loop.ts @@ -236,11 +236,11 @@ export function archiveUnder(run: RunState, path: string): void { /** A pass of the run, current or archived, with the path of the instance it belongs to. */ export type RunPass = StepPass & { step: string; within?: { step: string; pass: number } }; -/** Every pass of the run, current and archived. */ -export function runPasses(run: RunState): RunPass[] { +/** Every pass of the run, current and archived, with each step's current pass as `current` reads it. */ +export function runPasses(run: RunState, currentPass: (state: StepState) => StepPass | undefined = passOf): RunPass[] { return [ ...run.steps.flatMap((state) => { - const current = passOf(state); + const current = currentPass(state); return [...state.passes ?? [], ...(current ? [current] : [])].map((pass) => ({ ...pass, step: state.id })); }), ...(run.archive ?? []), diff --git a/src/workflow/parse.ts b/src/workflow/parse.ts index 801831928..ff4fbfcb2 100644 --- a/src/workflow/parse.ts +++ b/src/workflow/parse.ts @@ -21,7 +21,7 @@ import { REQUIREMENT_KEYS, RequirementsSchema, SetupSchema, type Requirements, t const HEADER_KEYS = new Set(['schemaVersion', 'name', 'description', 'inputs', 'defaults', 'ends', 'outputs', 'loop', 'timeout', 'budget', 'setup', 'steps']); const ACTION_KEYS = Object.keys(STEP_ACTIONS); -const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session', 'scorers', 'requires', 'timeout', 'budget', 'max_turns']); +const STEP_KEYS = new Set(['id', 'needs', 'if', ...ACTION_KEYS, 'env', 'with', 'agent', 'model', 'output', 'end', 'loop', 'break', 'for_each', 'approve', 'choose', 'interactive', 'workspace', 'session', 'scorers', 'requires', 'timeout', 'budget', 'max_turns', 'deliver']); const STEP_LOOP_KEYS = new Set(['back_to', 'while', 'max_iterations']); const FOR_EACH_KEYS = new Set(['items', 'max_parallel']); const WORKFLOW_LOOP_KEYS = new Set(['while', 'max_iterations']); @@ -523,6 +523,20 @@ function parseBody( }); } const sessionPair = item.items.find((pair) => scalarString(pair.key) === 'session'); + const deliverPair = item.items.find((pair) => scalarString(pair.key) === 'deliver'); + const deliver = scalarString(deliverPair?.value)?.trim(); + // Only what no enclosing step can change is checked per file. A prompt without workspace may continue the copy of + // the step that runs its workflow, so workflow start checks it on the whole tree. + const undeliverable = deliver !== 'pr' ? `${label} deliver must be pr; TeamAI opens a pull request and never merges` + : workspace === 'read-only' ? `${label} has deliver: pr but runs read-only; set workspace: new so it has a branch to deliver` + : commandPair && workspace === undefined ? `${label} has deliver: pr but runs read-only, as a command without workspace does; set workspace: new so it has a branch to deliver` + : undefined; + if (deliverPair && undeliverable) { + diagnostics.push({ + ...locate(file, lineCounter, deliverPair.value ?? deliverPair.key ?? item, `${field}.deliver`, undeliverable), + code: 'bad_delivery', + }); + } let session: WorkflowStep['session']; if (sessionPair) { const sessionText = scalarString(sessionPair.value)?.trim(); @@ -574,6 +588,7 @@ function parseBody( ...(breakSource !== undefined ? { break: breakSource } : {}), ...(forEach ? { forEach } : {}), ...(workspace ? { workspace } : {}), + ...(deliver === 'pr' ? { deliver } : {}), ...(session ? { session } : {}), }); } diff --git a/src/workflow/pr-delivery.ts b/src/workflow/pr-delivery.ts new file mode 100644 index 000000000..d259bb9a2 --- /dev/null +++ b/src/workflow/pr-delivery.ts @@ -0,0 +1,339 @@ +/** + * Pull request delivery: what it reads of the business repository and of its host, and nothing it writes to a run. The + * engine builds a job under the run lock, calls `deliverPr` once it released the lock, and applies the outcome under + * the lock again, so no network call holds the lock and this module needs nothing of the engine's. + */ +import os from 'node:os'; +import { providerForRemote, type OriginProvider } from '../providers/registry.js'; +import { NON_INTERACTIVE_GIT_ENV } from '../utils/git-env.js'; +import type { PullRequestRef } from '../providers/types.js'; +import { sanitizePrUrl, type PrUrl } from './pr-url.js'; +import type { DeliveryKey, UnresolvedCreate } from './delivery-schema.js'; +import type { DeliveryFailure } from './step-state.js'; +import { fetchMissingCommits, git, protectedPaths } from './worktree.js'; + +/** Everything a delivery needs, read from the run under its lock. */ +export interface DeliveryJob { + key: DeliveryKey; + runId: string; + runName: string; + /** The checkout the run started or was claimed in, which reads the delivered branch. */ + checkout: string; + /** The run base commit the delivered branch is compared with. */ + base: string; + /** The latest commit a report recorded on this branch, read under the run lock. */ + recordedHead?: string; + /** Work steps inside the delivering one left on branches no step delivers, which the delivered branch must hold. */ + childWork: Array<{ step: string; branch: string; commit: string }>; + /** The workflow step whose copy the branch is, when another step's: retrying it runs its steps again in a new branch. */ + openedBy?: string; + /** The branch the pull request goes into. */ + target: string; + /** A URL the step's driver reported and the engine kept: nothing is opened for it. */ + reported?: PrUrl; +} + +/** Why a delivery opened no pull request. `no_checkout` is found under the run lock, before there is a job. */ +export type DeliveryFault = + | { kind: 'no_checkout' } + | { kind: 'unreadable' } + | { kind: 'protected_path'; files: string[] } + /** Only the commits a later report recorded after the step's own head change `files`. */ + | { kind: 'later_protected_path'; files: string[] } + | { kind: 'branch_moved' } + /** Origin's branch holds the recorded commit and more, at `tip`: a report of `tip` may still be on its way. */ + | { kind: 'branch_ahead'; tip: string } + /** The recorded commit origin's branch is at lacks the step's head: a later step rewrote the branch's history. */ + | { kind: 'branch_rewritten' } + /** Origin's branch lacks the work step `step` inside the delivering one left on `branch`. */ + | { kind: 'child_work_left_out'; step: string; branch: string } + | { kind: 'branch_missing' } + | { kind: 'branch_unreadable' } + | { kind: 'no_provider' } + | { kind: 'lookup_failed'; provider: string } + | { kind: 'create_failed'; provider: string } + | { kind: 'create_unconfirmed'; provider: string } + | { kind: 'unsafe_url'; provider: string } + | { kind: 'withdrawn' }; + +export type DeliveryOutcome = { ok: true; pr: PrUrl } | { ok: false; fault: DeliveryFault }; + +/** + * Asked under the run lock right before a pull request is opened: `open` while the step still delivers the job, whose + * request the run then lists as unresolved; `refused` when it no longer does; `busy` when nobody knows yet, because the + * run lock could not be taken, a report recorded another commit on the branch since the job was picked, or a step that + * writes the branch is leased. Only `open` opens anything. + */ +export type CreateGate = () => Promise<'open' | 'refused' | 'busy'>; + +/** How long a host may take to answer one lookup or one create. */ +export const DELIVERY_TIMEOUT_MS = 30_000; + +/** How delivery runs git that reaches origin: bounded, and never waiting on a prompt. */ +const ORIGIN_GIT = { timeoutMs: DELIVERY_TIMEOUT_MS, env: NON_INTERACTIVE_GIT_ENV }; + +/** A full object id: the head comes from a driver's report, and anything else could reach git as an option. */ +const OBJECT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/i; + +const failed = (fault: DeliveryFault): DeliveryOutcome => ({ ok: false, fault }); + +/** + * Delivers one branch. Its changes against the run base are checked before anything else, a reported URL included, + * so no pull request goes out, and no step succeeds, with a protected path changed. Then the open request of the branch + * is looked up, and only when there is none, origin's branch is still at the checked commit, and `gate` opens, is one + * opened; then it is looked up again. `busy` is nothing opened and no outcome: the branch moved while the lookup ran, or + * the gate could not answer, and the job is for a later call, which checks it again. + * Nothing here logs in, prompts or merges, and no text a host answered with is kept, since it may hold a credential. + */ +export async function deliverPr(job: DeliveryJob, gate: CreateGate): Promise { + if (!OBJECT_ID.test(job.key.head) || !OBJECT_ID.test(job.base)) return failed({ kind: 'unreadable' }); + const tip = await originTip(job.checkout, job.key.branch); + if (tip.kind === 'unreadable') return failed({ kind: 'branch_unreadable' }); + if (tip.commit === undefined) return failed({ kind: 'branch_missing' }); + if (tip.commit !== job.key.head && tip.commit !== job.recordedHead) { + const ahead = await contains(job, { commit: tip.commit, ancestor: job.recordedHead ?? job.key.head }); + if ('fault' in ahead) return failed(ahead.fault); + return failed(ahead.contains ? { kind: 'branch_ahead', tip: tip.commit } : { kind: 'branch_moved' }); + } + if (tip.commit !== job.key.head) { + const holds = await contains(job, { commit: tip.commit, ancestor: job.key.head }); + if ('fault' in holds) return failed(holds.fault); + if (!holds.contains) return failed({ kind: 'branch_rewritten' }); + } + const changed = await changedFiles({ ...job, key: { ...job.key, head: tip.commit } }); + if ('fault' in changed) return failed(changed.fault); + const files = protectedPaths(changed.files); + if (files.length > 0 && tip.commit !== job.key.head) { + const own = await changedFiles(job); + if ('fault' in own) return failed(own.fault); + const owned = protectedPaths(own.files); + return failed(owned.length > 0 ? { kind: 'protected_path', files: owned } : { kind: 'later_protected_path', files }); + } + if (files.length > 0) return failed({ kind: 'protected_path', files }); + for (const child of job.childWork) { + const held = await contains(job, { commit: tip.commit, ancestor: child.commit }); + if ('fault' in held) return failed(held.fault); + if (!held.contains) return failed({ kind: 'child_work_left_out', step: child.step, branch: child.branch }); + } + if (job.reported) return { ok: true, pr: job.reported }; + const origin = await originOf(job.checkout); + if (origin.kind === 'transport') return failed({ kind: 'no_provider' }); + const { provider, repo } = origin; + const ref: PullRequestRef = { repo, source: job.key.branch, target: job.target, timeoutMs: DELIVERY_TIMEOUT_MS }; + const found = await findOpen(origin, ref); + if (found.kind === 'failed') return failed({ kind: 'lookup_failed', provider: provider.name }); + if (found.kind === 'unsafe') return failed({ kind: 'unsafe_url', provider: provider.name }); + // A step of the branch may have pushed while the lookup ran, so neither a found request nor a new one is taken on the + // commit checked before it. A push after this read shows in the pull request itself, and nothing here closes that window. + const checked = await originTip(job.checkout, job.key.branch); + if (checked.kind === 'unreadable') return failed({ kind: 'branch_unreadable' }); + if (checked.commit !== tip.commit) return 'busy'; + if (found.kind === 'found') return { ok: true, pr: found.pr }; + const gated = await gate(); + if (gated === 'busy') return 'busy'; + if (gated === 'refused') return failed({ kind: 'withdrawn' }); + let answered = false; + try { + await provider.createPullRequest({ + repo, source: job.key.branch, target: job.target, title: `${job.runName}: ${job.key.step}`, + description: `Opened by TeamAI for review: workflow ${job.runName}, run ${job.runId}, step ${job.key.step}, pass ${job.key.pass}.\n\nBase commit: ${job.base}\n\nTeamAI never merges this pull request.`, + // Run outside the member checkout, so the provider's CLI can neither push from it nor read its branch. + cwd: os.tmpdir(), + spawnTimeoutMs: DELIVERY_TIMEOUT_MS, + }); + answered = true; + } catch { + // A create that failed or timed out may still have opened the request, so the lookup below decides. + } + // One that answered may name the request by another URL than the lookup a later delivery of the branch finds it by, + // or before the host lists it, so only the lookup's URL is kept: one request, one receipt. + const opened = await findOpen(origin, ref); + if (opened.kind === 'found') return { ok: true, pr: opened.pr }; + if (opened.kind === 'unsafe') return failed({ kind: 'unsafe_url', provider: provider.name }); + return failed({ kind: answered || opened.kind === 'failed' ? 'create_unconfirmed' : 'create_failed', provider: provider.name }); +} + +/** + * Why a URL a driver reported for `branch` is not kept: on the origin's host it must be the open request of the branch + * into `target`, as the provider's lookup answers, so one the lookup cannot confirm is not kept either; delivery's own + * lookup still finds the open request. A URL on another host, such as a private review service, or an origin only git + * reaches, keeps it, and delivery accepts it as it is. + */ +export async function reportedPrMismatch( + { checkout, url, branch, target }: { checkout: string; url: PrUrl; branch: string; target: string }, +): Promise { + const origin = await originOf(checkout); + if (origin.kind === 'transport' || new URL(url).hostname !== origin.hostname) return undefined; + const found = await findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); + if (found.kind === 'none') return `branch ${branch} has no open pull request into ${target}`; + if (found.kind !== 'found') return `could not be checked against the open pull requests of branch ${branch} into ${target}`; + return found.pr === url ? undefined : `is not the open pull request of branch ${branch} into ${target}`; +} + +/** + * Whether the host holds the open request `create` asked for, by lookup alone: nothing of the branch is read, and + * nothing is opened. `failed` is a host that cannot answer or an origin no provider reads. `unsafe` is a URL that cannot be kept. + */ +export async function findDelivered( + checkout: string, { branch, target }: Pick, +): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' } | { kind: 'unsafe' }> { + const origin = await originOf(checkout); + if (origin.kind === 'transport') return { kind: 'failed' }; + return findOpen(origin, { repo: origin.repo, source: branch, target, timeoutMs: DELIVERY_TIMEOUT_MS }); +} + +/** The provider of the checkout's configured origin; git alone when the checkout has none. */ +async function originOf(checkout: string): Promise { + const remote = await git(checkout, ['config', '--get', 'remote.origin.url']); + return remote.code === 0 ? providerForRemote(remote.stdout) : { kind: 'transport' }; +} + +/** + * What a lookup said: the open request, sanitized; none; no answer; or a URL that cannot be kept, which a URL on another + * host than the origin's pull requests is, whichever provider answered. + */ +async function findOpen( + { pullRequests, hostname }: Extract, ref: PullRequestRef, +): Promise<{ kind: 'found'; pr: PrUrl } | { kind: 'none' } | { kind: 'failed' } | { kind: 'unsafe' }> { + let url: string | undefined; + try { + url = await pullRequests.findOpen(ref); + } catch { + return { kind: 'failed' }; + } + if (url === undefined) return { kind: 'none' }; + const clean = sanitizePrUrl(url); + return clean.url && new URL(clean.url).hostname === hostname ? { kind: 'found', pr: clean.url } : { kind: 'unsafe' }; +} + +/** The commit origin's `branch` is at, read by the checkout; none when origin has no such branch. */ +async function originTip(checkout: string, branch: string): Promise<{ kind: 'read'; commit?: string } | { kind: 'unreadable' }> { + const ref = `refs/heads/${branch}`; + const listed = await git(checkout, ['-c', 'credential.interactive=false', 'ls-remote', 'origin', ref], ORIGIN_GIT); + if (listed.code !== 0) return { kind: 'unreadable' }; + const commit = listed.stdout.split('\n').map((line) => line.split('\t')).find(([, name]) => name?.trim() === ref)?.[0]; + return commit ? { kind: 'read', commit } : { kind: 'read' }; +} + +/** The files the delivered head changes against the run base, read in the run's checkout once it fetched what it lacks. */ +async function changedFiles({ key, runId, checkout, base }: DeliveryJob): Promise<{ files: string[] } | { fault: DeliveryFault }> { + if (!OBJECT_ID.test(key.head) || !OBJECT_ID.test(base)) return { fault: { kind: 'unreadable' } }; + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [base, key.head] }, ORIGIN_GIT); + if (fetched.missing.length > 0) return { fault: { kind: 'unreadable' } }; + const diff = await git(checkout, ['diff', '--no-renames', '--name-only', '-z', base, key.head, '--']); + if (diff.code !== 0) return { fault: { kind: 'unreadable' } }; + return { files: diff.stdout.split('\0').filter(Boolean) }; +} + +/** Whether `commit` holds `ancestor`, read in the run's checkout once it fetched what it lacks. */ +async function contains({ runId, checkout }: DeliveryJob, { commit, ancestor }: { commit: string; ancestor: string }): Promise<{ contains: boolean } | { fault: DeliveryFault }> { + if (!OBJECT_ID.test(commit) || !OBJECT_ID.test(ancestor)) return { contains: false }; + const fetched = await fetchMissingCommits(checkout, { run: runId, commits: [ancestor, commit] }, ORIGIN_GIT); + if (fetched.missing.includes(commit)) return { fault: { kind: 'unreadable' } }; + // A commit the checkout holds brings its history, so an ancestor still missing is not in it. + if (fetched.missing.length > 0) return { contains: false }; + const { code } = await git(checkout, ['merge-base', '--is-ancestor', ancestor, commit]); + return code === 0 || code === 1 ? { contains: code === 0 } : { fault: { kind: 'unreadable' } }; +} + +const LISTED_FILES = 20; + +function listedFiles(files: string[]): string { + const more = files.length > LISTED_FILES ? `, and ${files.length - LISTED_FILES} more` : ''; + return `${files.slice(0, LISTED_FILES).join(', ')}${more}`; +} + +/** The failure a step keeps for `fault`: what happened, where the work is, and what to do next. */ +export function deliveryFailure({ runId, key, recordedHead, openedBy }: Pick, fault: DeliveryFault): DeliveryFailure { + const { step, branch } = key; + const head = recordedHead ?? key.head; + const retry = `teamai workflow retry ${runId} ${step}`; + switch (fault.kind) { + case 'no_checkout': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the run records no checkout, or no base branch, to deliver branch ${branch} from and into. The work is kept in branch ${branch}. Open a pull request from it yourself, or start the run again from a checkout of its repository.`, + }; + case 'unreadable': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the run's checkout cannot read the commit branch ${branch} reported, or the run base, even after fetching the run's branches from origin, so its changes cannot be checked. Push branch ${branch} to origin, then run: ${retry}`, + }; + case 'protected_path': + return { + code: 'protected_path', + message: `Step "${step}" changed protected paths against the run base: ${listedFiles(fault.files)}. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. Have the step leave those paths alone, then run: ${retry}`, + }; + case 'later_protected_path': { + const rerun = openedBy ? `. Or, once workflow step "${openedBy}" has stopped, run its steps again in a new branch: teamai workflow retry ${runId} ${openedBy}` : ''; + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: commit ${head}, which a later report recorded on the shared branch ${branch}, changed protected paths against the run base: ${listedFiles(fault.files)}. Step "${step}" itself left them alone, so a retry does not run it again. TeamAI delivers no pull request that changes CI, hooks or agent configuration. The work is kept on origin in branch ${branch}. To deliver the step's own work without the later commits, put branch ${branch} on origin back at ${key.head}, the commit the step reported, then run: ${retry}${rerun}`, + }; + } + case 'branch_missing': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: origin has no branch ${branch}; push ${head} to it, then run: ${retry}`, + }; + case 'branch_ahead': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: branch ${branch} on origin is at commit ${fault.tip}, which holds commit ${head}, the last one a report recorded on it, and later commits no report recorded, so TeamAI has not checked them, and no pull request was opened or accepted. Put branch ${branch} on origin back at ${head}, then run: ${retry}`, + }; + case 'branch_moved': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: branch ${branch} on origin moved away from commit ${head}, the last one a report recorded on it, so it no longer holds the changes TeamAI checked, and no pull request was opened or accepted. Put branch ${branch} on origin back at ${head}, then run: ${retry}`, + }; + case 'branch_rewritten': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: commit ${head}, the last one a report recorded on branch ${branch}, does not contain commit ${key.head} the step reported, so a pull request from it would not hold the step's work, and none was opened or accepted. Have a step on that branch record a commit that contains both, or put branch ${branch} on origin back at ${key.head}, then run: ${retry}`, + }; + case 'child_work_left_out': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: step "${fault.step}" wrote branch ${fault.branch}, which branch ${branch} does not hold, so a pull request from it would leave that work out, and none was opened or accepted. The work is kept in branches ${branch} and ${fault.branch}. Have a later step in the workflow need step "${fault.step}", so it merges branch ${fault.branch}, or give step "${fault.step}" its own deliver: pr. A run keeps the workflow it started with, so start the run again after you change the workflow.`, + }; + case 'branch_unreadable': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: TeamAI could not read branch ${branch} on origin to check that it still holds commit ${key.head} the step reported, so no pull request was opened or accepted. Check the checkout's access to origin, then run: ${retry}`, + }; + case 'no_provider': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: TeamAI cannot open a pull request on this repository's origin. The work is kept on origin in branch ${branch}. Open a pull request from it, or have the driver report changes.pr_url, then run: ${retry}`, + }; + case 'lookup_failed': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the ${fault.provider} provider could not say whether branch ${branch} already has an open pull request, so TeamAI opened none. Check its credentials and its access to the repository. The work is kept on origin in branch ${branch}. Then run: ${retry}`, + }; + case 'create_failed': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the ${fault.provider} provider did not open a pull request from branch ${branch}. Check its credentials, its access to the repository, and that the branch is on origin. The work is kept in branch ${branch}. Then run: ${retry}`, + }; + case 'create_unconfirmed': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the ${fault.provider} host may have opened a pull request from branch ${branch}, but TeamAI could not confirm it. No receipt was recorded. Check its credentials and its access to the repository, then run: ${retry}`, + }; + case 'unsafe_url': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" failed: the ${fault.provider} provider answered with a pull request URL TeamAI cannot keep. The work is kept on origin in branch ${branch}. Check the open pull requests of that branch, then run: ${retry}`, + }; + case 'withdrawn': + return { + code: 'delivery_failed', + message: `PR delivery for step "${step}" stopped: the step no longer delivered when its pull request was about to be opened, so none was. The work is kept in branch ${branch}.`, + }; + default: { + const unknown: never = fault; + return unknown; + } + } +} diff --git a/src/workflow/pr-url.ts b/src/workflow/pr-url.ts new file mode 100644 index 000000000..53e47dd22 --- /dev/null +++ b/src/workflow/pr-url.ts @@ -0,0 +1,40 @@ +import { z } from 'zod'; + +const MAX_LENGTH = 512; + +/** + * A pull request URL safe to store and publish: https on any host and path, no credentials, no query or fragment, at + * most 512 characters. Parsing a sanitized value gives it back unchanged, so a stored URL parses to itself. + */ +export const PrUrlSchema = z.string().transform((value, ctx) => { + const clean = sanitize(value); + if ('reason' in clean) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: clean.reason }); + return z.NEVER; + } + return clean.url; +}).brand<'PrUrl'>(); +export type PrUrl = z.infer; + +function sanitize(value: string): { url: string } | { reason: string } { + if (value.length > MAX_LENGTH) return { reason: `exceeds ${MAX_LENGTH} characters` }; + let url: URL; + try { + url = new URL(value); + } catch { + return { reason: 'is not a valid URL' }; + } + if (url.protocol !== 'https:') return { reason: 'must use HTTPS' }; + if (url.username || url.password) return { reason: 'contains credentials' }; + url.search = ''; + url.hash = ''; + const sanitized = url.toString(); + return sanitized.length <= MAX_LENGTH ? { url: sanitized } : { reason: `exceeds ${MAX_LENGTH} characters` }; +} + +/** Why a reported URL was not kept is said without the value, which may hold a token. */ +export function sanitizePrUrl(value: unknown): { url: PrUrl; reason?: never } | { url?: never; reason: string } { + if (typeof value !== 'string') return { reason: 'must be a URL string' }; + const parsed = PrUrlSchema.safeParse(value); + return parsed.success ? { url: parsed.data } : { reason: parsed.error.issues[0]?.message ?? 'is not a valid URL' }; +} diff --git a/src/workflow/records.ts b/src/workflow/records.ts index c630d610d..e2e1d2c14 100644 --- a/src/workflow/records.ts +++ b/src/workflow/records.ts @@ -40,6 +40,7 @@ const RECORDED_FAILURES = [ 'tool_failed', 'tool_unavailable', 'agent_unavailable', 'model_unavailable', 'read_only_dirty', 'tool_unconfigured', 'tool_unsupported', 'merge_conflict', 'base_config', 'push_failed', 'workspace_base', 'bad_session', ...LIMIT_FAILURES, + 'delivery_failed', 'protected_path', ] as const; type RecordedFailure = (typeof RECORDED_FAILURES)[number]; @@ -101,6 +102,8 @@ export type WorkflowRunRecord = { stepCounts?: { total: number; succeeded: number; failed: number; skipped: number; cancelled: number }; /** Settled attempts across the run — the cumulative iteration count. */ iterations?: number; + /** The pull requests the run delivered, in the order they first appeared. Public on a public team repo; absent when it delivered none. */ + prs?: string[]; username: string; publishedAt: string; }; @@ -215,6 +218,8 @@ export function buildEndRecord( ]), ...(run.archive ?? []).map((archived) => entry(archived.step, archived.pass, archived)), ]; + // Receipts are what delivery recorded, never a step's outputs, and each URL appears once. + const prs = (run.deliveries ?? []).map((receipt) => receipt.url); return { ...buildStartRecord(run, driver, username, now), phase: 'end', @@ -236,6 +241,7 @@ export function buildEndRecord( // Cumulative when present; runs from before the counter fall back to the // bounded history length (their true count is unknowable now). iterations: run.settledCount ?? run.settledAttempts.length, + ...(prs.length ? { prs } : {}), }; } @@ -253,6 +259,7 @@ export function mergeRecord(origin: WorkflowRunRecord | null, next: WorkflowRunR const merged = { ...origin, ...next, startedAt: origin.startedAt }; if (next.end === undefined) delete merged.end; if (next.failure === undefined) delete merged.failure; + if (next.prs === undefined) delete merged.prs; return merged; } diff --git a/src/workflow/run-controls.ts b/src/workflow/run-controls.ts index c98cbb07c..400192ff4 100644 --- a/src/workflow/run-controls.ts +++ b/src/workflow/run-controls.ts @@ -10,7 +10,7 @@ import { archiveUnder, isUnder } from './loop.js'; import { spentTokens, startLimits } from './limits.js'; import { publishRunRecords, removeQueuedEndRecord } from './records.js'; import { contextOf, localId, statesIn } from './scope.js'; -import { answerHitl, cancel, repend, resetIteration, resetPending, settled, start, stopAttempt, stopWhere, stopped, type HitlAnswer, type StepState } from './step-state.js'; +import { answerHitl, cancel, cancellable, redeliver, repend, resetIteration, resetPending, settled, start, stopAttempt, stopWhere, stopped, type HitlAnswer, type StepState } from './step-state.js'; import type { RunState, WorkflowEnvelope } from './types.js'; async function controlRun( @@ -121,7 +121,7 @@ export async function cancelRun(config: LocalConfig, runId: string, secrets: Rec } else { const state = run.steps.find((state) => state.id === step); if (!state) return missingStepError(run, step); - if (state.status !== 'pending' && state.status !== 'waiting' && state.status !== 'leased' && state.status !== 'running') return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased or running steps can be cancelled`); + if (!cancellable(state)) return error('not_cancellable', `Step "${state.id}" is ${state.status}; only pending, waiting, leased, running or delivering steps can be cancelled`); const located = locateStep(run, step); if (!located) return missingStepError(run, step); changed = state.status === 'running' ? stopWhere(run, isUnder(state.id), failure, run.revision, now) : []; @@ -230,6 +230,7 @@ function retryInstance(run: RunState, state: StepState, now: string): void { cancel(state, failure, run.revision); break; case 'running': + case 'delivering': stopWhere(run, isUnder(state.id), failure, run.revision, now); break; case 'succeeded': @@ -273,7 +274,9 @@ export async function retryStep(config: LocalConfig, runId: string, secrets: Rec const reopened = run.status !== 'running'; for (const container of containers) { if (container.child === target) { - retryInstance(run, target, now); + // A failed delivery delivers the same branch again; the agent's work it delivers does not change. + if (target.status === 'failed' && target.failure?.code === 'delivery_failed' && target.delivery) redeliver(target, target.delivery); + else retryInstance(run, target, now); resetIteration(target); } for (const state of container.states) { diff --git a/src/workflow/scoring-evidence.ts b/src/workflow/scoring-evidence.ts index 3e2a3c0c5..235c0704c 100644 --- a/src/workflow/scoring-evidence.ts +++ b/src/workflow/scoring-evidence.ts @@ -13,7 +13,7 @@ import { import type { RunSubject } from '../scorers/label.js'; import { firstSettled, isUnder, passRange, passesWithin, runPasses } from './loop.js'; import { stepDefinitions } from './scope.js'; -import type { StepPass } from './step-state.js'; +import { agentPassOf, type StepPass } from './step-state.js'; import type { RunState } from './types.js'; import { branchName } from './workspace.js'; import { stepCheckout } from './worktree.js'; @@ -29,7 +29,7 @@ function ranIn(run: RunState, pass: StepPass): string | undefined { export function evidencePlan(run: RunState, subject: RunSubject) { const definitionOf = stepDefinitions(run); - const all = runPasses(run); + const all = runPasses(run, agentPassOf); const own = all.find((pass) => pass.step === subject.step && pass.pass === subject.pass); const workflow = definitionOf(subject.step)?.workflow !== undefined; const gaps: EvidenceGap[] = []; diff --git a/src/workflow/scoring.ts b/src/workflow/scoring.ts index 480e22331..4b46d1fea 100644 --- a/src/workflow/scoring.ts +++ b/src/workflow/scoring.ts @@ -35,6 +35,7 @@ import { error, ok } from './engine.js'; import { localDriver } from './local-driver.js'; import { firstSettled, passesWithin, runPasses, type RunPass } from './loop.js'; import { stepDefinitions } from './scope.js'; +import { agentPassOf, agentSucceeded } from './step-state.js'; import { parseStepPath } from './step-path.js'; import { installOwner, isRunId, readRun, runsDir } from './store.js'; import type { InlineWorkflow, RunState, WorkflowDefinition, WorkflowEnvelope, WorkflowStep } from './types.js'; @@ -50,12 +51,15 @@ function settledHere(run: RunState, pass: RunPass, install: string): boolean { return pass.installId === undefined ? run.publication === undefined : pass.installId === install; } -/** The passes a judge can judge: succeeded passes of prompt and workflow steps, and of a `for_each` step only its items. */ +/** + * The passes a judge can judge: passes of prompt and workflow steps whose agent succeeded, delivering or not, and of a + * `for_each` step only its items. + */ function judgeablePasses(run: RunState): Array<{ pass: RunPass; definition: WorkflowStep }> { const definitionOf = stepDefinitions(run); - return runPasses(run).flatMap((pass) => { + return runPasses(run, agentPassOf).flatMap((pass) => { const definition: WorkflowStep | undefined = definitionOf(pass.step); - if (pass.outcome !== 'success' || !definition) return []; + if (!agentSucceeded(pass) || !definition) return []; if (definition.forEach && parseStepPath(pass.step).at(-1)?.index === undefined) return []; return [{ pass, definition }]; }); @@ -67,7 +71,7 @@ function judgeablePasses(run: RunState): Array<{ pass: RunPass; definition: Work * stamp. Sampling is left to the caller. */ export function scoredPasses(run: RunState, install: string): ScorablePass[] { - const all = runPasses(run); + const all = runPasses(run, agentPassOf); const pinned = run.scorers ?? {}; return judgeablePasses(run).flatMap(({ pass, definition }) => { const scorers = (definition.scorers?.names ?? []).filter((name) => Object.hasOwn(pinned, name)); diff --git a/src/workflow/snapshot.ts b/src/workflow/snapshot.ts index d057d5dcb..c99a163b3 100644 --- a/src/workflow/snapshot.ts +++ b/src/workflow/snapshot.ts @@ -5,6 +5,7 @@ import { z } from 'zod'; import { canonicalJson } from '../utils/canonical-json.js'; import { redact } from '../utils/redact.js'; import { isRecord } from './expression.js'; +import { CommitIdSchema, DeliveryReceiptSchema, UnresolvedCreateSchema } from './delivery-schema.js'; import { ArchivedPassSchema, LimitsFromSchema, ReleasedTokensSchema, SessionContinuationSchema, StepCheckpointSchema, type ArchivedPass, type HitlAnswer, type StepPass, type StepState, } from './step-state.js'; @@ -19,9 +20,6 @@ import type { RunState, StepLease, WorkflowDefinition, WorkflowFailure } from '. const OwnerSchema = z.object({ id: z.string().min(1), user: z.string() }); export type HandoffOwner = z.infer; -/** A claim hands every commit a snapshot names to git, so each is only ever a full commit id. */ -const CommitIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); - const ReportedCommitsSchema = z.object({ changes: z.object({ base: CommitIdSchema, from: CommitIdSchema, to: CommitIdSchema, commits: z.array(CommitIdSchema) }).optional(), read: z.object({ base: CommitIdSchema, commit: CommitIdSchema }).optional(), @@ -53,7 +51,10 @@ const BodySchema = z.object({ round: z.number().int().optional(), base: z.object({ ref: z.string().optional(), commit: CommitIdSchema }).optional(), /** The run's workspaces by branch, with the commit each was last reported at. Which driver holds one stays local. */ - copies: z.record(z.object({ commit: CommitIdSchema.optional() })), + copies: z.record(z.object({ commit: CommitIdSchema.optional(), from: z.string().optional() })), + deliveries: z.array(DeliveryReceiptSchema).optional(), + /** The requests a claimer looks up before anything opens another: the machine that handed the run over may have opened them. */ + creating: z.array(UnresolvedCreateSchema).optional(), sessionContinuations: z.array(SessionContinuationSchema).optional(), releasedTokens: z.array(ReleasedTokensSchema).optional(), /** The scorers the run pinned, by hash only: the machine that claims the run uses its own copy when the hash matches. */ @@ -75,8 +76,12 @@ const SnapshotSchema = z.object({ write: z.string().min(1), }).merge(BodySchema).superRefine((snapshot, ctx) => { // A claim hands each copy's branch to git, so a snapshot never names a branch outside its own run. - for (const branch of Object.keys(snapshot.copies)) { + for (const [branch, copy] of Object.entries(snapshot.copies)) { if (!branch.startsWith(`teamai/${snapshot.run}/`)) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['copies', branch], message: 'is not a branch of this run' }); + // A pull request from the copy may target the branch it started from. + if (copy.from !== undefined && !copy.from.startsWith(`teamai/${snapshot.run}/`)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['copies', branch, 'from'], message: 'is not a branch of this run' }); + } } }); export type RunSnapshot = z.infer; @@ -193,8 +198,8 @@ const RUN_RULES = { end: 'publish', failure: 'rewrite', inputs: 'publish', sourceHash: 'publish', teamRevision: 'publish', definition: 'rewrite', workflows: 'rewrite', scorers: 'rewrite', steps: 'rewrite', settledAttempts: 'rewrite', settledCount: 'rewrite', endRecordQueued: 'drop', cancelledAttempts: 'rewrite', archive: 'rewrite', round: 'publish', base: 'publish', - checkout: 'drop', copies: 'rewrite', sessionContinuations: 'publish', releasedTokens: 'publish', stoppedAttempts: 'drop', publication: 'drop', createdAt: 'publish', limitsFrom: 'publish', - updatedAt: 'publish', + checkout: 'drop', copies: 'rewrite', deliveries: 'publish', creating: 'publish', sessionContinuations: 'publish', releasedTokens: 'publish', + stoppedAttempts: 'drop', publication: 'drop', createdAt: 'publish', limitsFrom: 'publish', updatedAt: 'publish', } satisfies FieldRules; const STEP_RULES = { @@ -202,15 +207,15 @@ const STEP_RULES = { iteration: 'publish', round: 'publish', status: 'publish', attempt: 'publish', lease: 'rewrite', outcome: 'publish', outputs: 'publish', end: 'publish', results: 'publish', settledAt: 'publish', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', - agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', unenforced: 'publish', expiresAt: 'publish', - limitsFrom: 'publish', + agent: 'publish', used: 'publish', installId: 'publish', loopFinal: 'publish', failure: 'rewrite', delivery: 'publish', unenforced: 'publish', + expiresAt: 'publish', limitsFrom: 'publish', } satisfies FieldRules; const PASS_RULES = { pass: 'publish', attempt: 'publish', outcome: 'publish', outputs: 'publish', end: 'publish', results: 'publish', failure: 'rewrite', exitCode: 'publish', message: 'drop', tool: 'publish', model: 'publish', session: 'publish', tokens: 'publish', changes: 'publish', read: 'publish', agent: 'publish', used: 'publish', installId: 'publish', answer: 'rewrite', - unenforced: 'publish', + delivery: 'publish', unenforced: 'publish', } satisfies FieldRules; const ARCHIVED_RULES = { ...PASS_RULES, step: 'publish', within: 'publish' } satisfies FieldRules; @@ -225,7 +230,7 @@ const FAILURE_RULES = { 'code': 'publish', message: 'rewrite' } satisfies FieldR // A lease another machine held is released when the run is handed over, so the session it offered stays here. const LEASE_RULES = { attempt: 'publish', revision: 'publish', driver: 'rewrite', session: 'drop', unenforced: 'drop', expiresAt: 'drop' } satisfies FieldRules; -const COPY_RULES = { owner: 'drop', commit: 'publish', used: 'drop' } satisfies FieldRules[string]>; +const COPY_RULES = { owner: 'drop', commit: 'publish', from: 'publish', used: 'drop' } satisfies FieldRules[string]>; const FAILURE_REWRITES = { message: () => ({ message: HANDED_FAILURE_MESSAGE }), diff --git a/src/workflow/step-state.ts b/src/workflow/step-state.ts index ab7bd054e..34572b4b3 100644 --- a/src/workflow/step-state.ts +++ b/src/workflow/step-state.ts @@ -2,6 +2,7 @@ import { z } from 'zod'; import { AgentObservationSchema, UsedSchema } from '../agent-observation.js'; import { DriverSchema } from '../driver-identity.js'; import type { TokenUsage } from '../tool-launch/stdout.js'; +import { PrUrlSchema } from './pr-url.js'; import { UnenforcedSchema } from './requirements.js'; import type { RunState, StepLease, WorkflowFailure } from './types.js'; import type { WorkspaceMode } from './workspace.js'; @@ -108,6 +109,13 @@ const AnswerSchema = z.object({ /** The answer and provenance retained for one pass of a step with HITL. */ export type HitlAnswer = z.infer; +/** + * What a step that delivers hands to delivery once its agent succeeded: the branch, the commit that branch must hold on + * origin, and a URL its driver reported and the engine kept. The target is not kept; it follows from the run. + */ +const DeliverySchema = z.object({ branch: z.string().min(1), head: z.string().min(1), reported: PrUrlSchema.optional() }); +export type Delivery = z.infer; + /** One earlier pass of a step instance that a loop ran again. */ export const StepPassSchema = z.object({ pass: z.number().int(), @@ -118,6 +126,8 @@ export const StepPassSchema = z.object({ results: ResultsSchema.optional(), failure: FailureSchema.optional(), ...report, + /** What the pass delivered once its agent succeeded, so a pass whose delivery failed or was cancelled is still the agent's success. */ + delivery: DeliverySchema.optional(), /** The answer retained with this archived pass. */ answer: AnswerSchema.optional(), }); @@ -159,6 +169,8 @@ const settledFields = { results: ResultsSchema.optional(), settledAt: z.number().int().default(0), attempt: z.string().optional(), + /** Kept once the step settles, so a retry of a failed delivery delivers the same branch again. */ + delivery: DeliverySchema.optional(), ...report, }; @@ -175,6 +187,8 @@ const WaitingSchema = StepBaseSchema.extend({ status: z.literal('waiting') }); * the workflow it runs counts from. Both are set when it starts. */ const RunningSchema = StepBaseSchema.extend({ status: z.literal('running'), expiresAt: z.string().optional(), limitsFrom: LimitsFromSchema.optional() }); +/** `delivering`: the agent succeeded and its work is on its branch; dependents wait until that branch has its pull request. */ +const DeliveringSchema = StepBaseSchema.extend({ status: z.literal('delivering'), ...settledFields, delivery: DeliverySchema }); const SucceededSchema = StepBaseSchema.extend({ status: z.literal('succeeded'), outcome: z.literal('success').default('success'), ...settledFields, loopFinal: z.boolean().optional(), }); @@ -182,40 +196,26 @@ const FailedSchema = StepBaseSchema.extend({ status: z.literal('failed'), outcome: z.literal('failed').default('failed'), ...settledFields, failure: FailureSchema.optional(), }); const SkippedSchema = StepBaseSchema.extend({ status: z.literal('skipped'), settledAt: z.number().int().default(0) }); -const CancelledSchema = StepBaseSchema.extend({ - status: z.literal('cancelled'), settledAt: z.number().int().default(0), attempt: z.string().optional(), failure: FailureSchema, unenforced: report.unenforced, +/** A step cancelled while it delivered keeps what its agent settled, as `delivering` holds it. */ +const CancelledSchema = StepBaseSchema.extend({ status: z.literal('cancelled'), ...settledFields, failure: FailureSchema }); + +const LeasedSchema = StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }); +/** + * A step as a run handed to another machine carries it. A leased step keeps its attempt and the kind of driver that + * holds it, not that driver's machine or workspace, which mean nothing on the machine that takes the run. + */ +const HandedLeaseSchema = StepBaseSchema.extend({ + status: z.literal('leased'), attempt: z.string(), lease: z.object({ attempt: z.string(), revision: z.number(), driverKind: z.string() }), }); +/** Every status but `leased` is the same locally and in a snapshot, so a status added here reaches both unions. */ +const SHARED_VARIANTS = [PendingSchema, WaitingSchema, RunningSchema, DeliveringSchema, SucceededSchema, FailedSchema, SkippedSchema, CancelledSchema] as const; -export const StepStateSchema = z.discriminatedUnion('status', [ - PendingSchema, - WaitingSchema, - StepBaseSchema.extend({ status: z.literal('leased'), attempt: z.string(), lease: LeaseSchema }), - RunningSchema, - SucceededSchema, - FailedSchema, - SkippedSchema, - CancelledSchema, -]); +export const StepStateSchema = z.discriminatedUnion('status', [LeasedSchema, ...SHARED_VARIANTS]); type ReadonlyId = T extends unknown ? Omit & { readonly id: string } : never; /** A step instance's `id` is read-only: the engine indexes instances by it, and every transition keeps it. */ export type StepState = ReadonlyId>; -/** - * A step as a run handed to another machine carries it. A leased step keeps its attempt and the kind of driver that - * holds it, not that driver's machine or workspace, which mean nothing on the machine that takes the run. - */ -export const StepCheckpointSchema = z.discriminatedUnion('status', [ - PendingSchema, - WaitingSchema, - StepBaseSchema.extend({ - status: z.literal('leased'), attempt: z.string(), lease: z.object({ attempt: z.string(), revision: z.number(), driverKind: z.string() }), - }), - RunningSchema, - SucceededSchema, - FailedSchema, - SkippedSchema, - CancelledSchema, -]); +export const StepCheckpointSchema = z.discriminatedUnion('status', [HandedLeaseSchema, ...SHARED_VARIANTS]); export type StepCheckpoint = ReadonlyId>; /** A lease another machine held when the run was handed over. */ export type HandedLease = Extract; @@ -225,6 +225,7 @@ type StepIn = Extract; type PendingStep = StepIn<'pending'>; export type LeasedStep = StepIn<'leased'>; type RunningStep = StepIn<'running'>; +export type DeliveringStep = StepIn<'delivering'>; type SucceededStep = StepIn<'succeeded'>; type FailedStep = StepIn<'failed'>; type SkippedStep = StepIn<'skipped'>; @@ -295,6 +296,7 @@ export function takeOver(step: StepCheckpoint, mode: WorkspaceMode | undefined): case 'pending': case 'waiting': case 'running': + case 'delivering': case 'succeeded': case 'failed': case 'skipped': @@ -325,6 +327,33 @@ export function succeed(state: StepState, settlement: Settlement) return become(state, { status: 'succeeded', outcome: 'success', ...settlement }); } +/** The agent's work is settled and durable; the step succeeds once `delivery` has its pull request. */ +export function startDelivery(state: StepState, settlement: Settlement, delivery: Delivery): DeliveringStep { + return become(state, { status: 'delivering', ...settlement, delivery }); +} + +/** A delivered step settles as its agent left it, at the revision the agent's report settled it. */ +export function delivered(state: DeliveringStep): SucceededStep { + const { id: _id, status: _status, ...settlement } = state; + return succeed(state, settlement); +} + +/** The failure codes only delivery gives a step, always after its agent succeeded. */ +const DELIVERY_FAILURE_CODES = ['delivery_failed', 'protected_path'] as const; +export type DeliveryFailure = { code: (typeof DELIVERY_FAILURE_CODES)[number]; message: string }; + +/** A delivery that failed stops the step, keeping what its agent did and the branch a retry delivers again. */ +export function deliveryFailed(state: DeliveringStep, failure: DeliveryFailure): FailedStep { + const { id: _id, status: _status, ...settlement } = state; + return fail(state, { ...settlement, message: failure.message, failure }); +} + +/** A failed delivery is retried in the same pass and attempt: the agent does not run again. */ +export function redeliver(state: FailedStep, delivery: Delivery): DeliveringStep { + const { id: _id, status: _status, outcome: _outcome, failure: _failure, message: _message, ...settlement } = state; + return startDelivery(state, settlement, delivery); +} + export function fail(state: StepState, settlement: Settlement): FailedStep { return become(state, { status: 'failed', outcome: 'failed', ...settlement }); } @@ -349,8 +378,16 @@ export function timeOut(state: LeasedStep | RunningStep, failure: WorkflowFailur return become(state, { status: 'failed', outcome: 'failed', settledAt, ...stoppedLease(state), message: failure.message, failure }); } -/** An open step is cancelled before it settles. */ -export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running'>, failure: WorkflowFailure, settledAt: number): CancelledStep { +/** + * An open step is cancelled before it settles. A leased one keeps the attempt it no longer settles, and what it did not + * enforce. A delivering one keeps what its agent settled, and what it delivered, since its agent succeeded whatever + * becomes of the delivery. + */ +export function cancel(state: StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'>, failure: WorkflowFailure, settledAt: number): CancelledStep { + if (state.status === 'delivering') { + const { id: _id, status: _status, ...settlement } = state; + return become(state, { ...settlement, status: 'cancelled', failure }); + } return become(state, { status: 'cancelled', settledAt, ...stoppedLease(state), failure }); } @@ -361,6 +398,22 @@ export function passOf(state: StepState): StepPass | undefined { return StepPassSchema.parse({ ...state, pass: state.pass ?? 1, outcome }); } +/** + * The current pass of a step as its agent left it, which scorers judge: a step that delivers ran its agent to success, + * so delivery never decides whether, or by which install, a pass is judged. + */ +export function agentPassOf(state: StepState): StepPass | undefined { + return state.status === 'delivering' ? StepPassSchema.parse({ ...state, pass: state.pass ?? 1, outcome: 'success' }) : passOf(state); +} + +/** Whether the agent of `pass` succeeded: it did, or only the delivery of what it wrote failed or was cancelled. */ +export function agentSucceeded(pass: StepPass): boolean { + const code: string | undefined = pass.failure?.code; + return pass.outcome === 'success' + || (pass.outcome === 'failed' && DELIVERY_FAILURE_CODES.some((delivery) => delivery === code)) + || (pass.outcome === 'cancelled' && pass.delivery !== undefined); +} + /** * A loop runs a settled step again: its current pass joins `passes`, and it waits as the next pass, with nothing of * the workflow it ran. `iteration` is the runs a loop owner has had so far. @@ -409,8 +462,51 @@ export function stopped(step: StepState): boolean { case 'waiting': case 'leased': case 'running': + case 'delivering': + case 'succeeded': + case 'skipped': + return false; + default: { + const unknown: never = step; + return unknown; + } + } +} + +/** A step a driver or the engine is still working on, so its workflow waits rather than being stuck. */ +export function inProgress(step: StepState): boolean { + switch (step.status) { + case 'waiting': + case 'leased': + case 'running': + case 'delivering': + return true; + case 'pending': + case 'succeeded': + case 'failed': + case 'skipped': + case 'cancelled': + return false; + default: { + const unknown: never = step; + return unknown; + } + } +} + +/** A step `teamai workflow cancel` may stop: one that has not settled. */ +export function cancellable(step: StepState): step is StepIn<'pending' | 'waiting' | 'leased' | 'running' | 'delivering'> { + switch (step.status) { + case 'pending': + case 'waiting': + case 'leased': + case 'running': + case 'delivering': + return true; case 'succeeded': + case 'failed': case 'skipped': + case 'cancelled': return false; default: { const unknown: never = step; @@ -432,6 +528,7 @@ export function settled(step: StepState | undefined): step is Extract boolean, failure skip(state, settledAt); break; case 'running': + case 'delivering': cancel(state, failure, settledAt); break; case 'leased': diff --git a/src/workflow/store.ts b/src/workflow/store.ts index 3de9777c9..887b6e648 100644 --- a/src/workflow/store.ts +++ b/src/workflow/store.ts @@ -42,6 +42,11 @@ export function runLockPath(config: LocalConfig, runId: string): string { return path.join(runsDir(config), `${runId}.lock`); } +/** Held while this machine delivers a run's pull requests: never the run lock, which no network call may hold. */ +export function deliveryLockPath(config: LocalConfig, runId: string): string { + return path.join(runsDir(config), `${runId}.delivery.lock`); +} + export function driverLockPath(config: LocalConfig, runId: string): string { return path.join(runsDir(config), runId, 'driver.lock'); } diff --git a/src/workflow/types.ts b/src/workflow/types.ts index 838d37d48..eb897bb47 100644 --- a/src/workflow/types.ts +++ b/src/workflow/types.ts @@ -1,6 +1,7 @@ /** Workflow types for the v1alpha1 local-driver subset. */ import type { PinnedScorer } from '../scorers/pin.js'; +import type { DeliveryReceipt, UnresolvedCreate } from './delivery-schema.js'; import type { ArchivedPass, LimitsFrom, ReleasedTokens, SessionContinuation, StepState, StoppedAttempt } from './step-state.js'; import type { Requirements, Setup, Unenforced } from './requirements.js'; import type { LeaseSession } from './workspace.js'; @@ -77,6 +78,8 @@ export interface WorkflowStep { * A prompt that omits it continues an enclosing `new` or `continue` copy, or stays read-only. */ workspace?: typeof WORKSPACE_MODES[number]; + /** Opens a review request after the branch's work succeeds. TeamAI never merges it. */ + deliver?: 'pr'; /** A session this step resumes. `continue` uses this step's previous pass. `from` names another step. */ session?: { continue: true } | { from: string }; /** What the driver that runs this step, or every step of the workflow it runs, must provide. */ @@ -214,8 +217,9 @@ export interface WorkflowDiagnostic { * to repeat or to run per item, or run on past the point the author wrote to stop at. * `bad_session`: a `session.from` pull rejects; the kept copy would resume a step the author did not name. * `bad_workspace`: `workspace` is not read-only, new, or continue; the kept copy would run the step in the wrong place. + * `bad_delivery`: delivery is not PR creation on a writable branch; the kept copy would apply a different policy. */ - code?: 'unknown_key' | 'missing_needs' | 'false_if' | 'bad_template' | 'dotted_name' | 'undeclared_output' | 'undeclared_end' | 'bad_loop' | 'bad_session' | 'bad_workspace' | 'workflow_namespace_conflict' | 'prompt_namespace_conflict'; + code?: 'unknown_key' | 'missing_needs' | 'false_if' | 'bad_template' | 'dotted_name' | 'undeclared_output' | 'undeclared_end' | 'bad_loop' | 'bad_session' | 'bad_workspace' | 'workflow_namespace_conflict' | 'prompt_namespace_conflict' | 'bad_delivery'; } export type StepOutcome = 'success' | 'failed' | 'released'; @@ -291,11 +295,23 @@ export interface RunState { checkout?: string; /** * Workspaces this run has handed out, by branch. A `continue` step stays with the driver that opened the copy. A copy a - * claim brought from another machine has no owner until a driver leases it. `used` is set once a step ran in the copy: - * a report on a lease of it was accepted, neither `released` nor a setup failure. Only a lease on an unused copy - * carries setup, and only an unused copy's owner gives it up when it hands its lease back. Local run state only. + * claim brought from another machine has no owner until a driver leases it. `from` is the run branch the copy started + * from, `after` items before other needs, so a pull request from it targets the nearest delivered branch it grew from. + * `used` is set once a step ran in the copy: a report on a lease of it was accepted, neither `released` nor a setup + * failure. Only a lease on an unused copy carries setup, and only an unused copy's owner gives it up when it hands its + * lease back. `used` is local run state only. */ - copies?: Record; + copies?: Record; + /** + * The pull requests delivered on this run, once each, in the order they first appeared. Append-only: a receipt says a + * request was opened, never that it is still open, so whether a delivery reuses one is its host's lookup to say. + */ + deliveries?: DeliveryReceipt[]; + /** + * The pull requests TeamAI asked a host to open and has no receipt for, listed under the run lock right before each + * create. A receipt or a lookup removes one; a retry, a cancel or an archive of its step does not. + */ + creating?: UnresolvedCreate[]; /** * Sessions a step resumed and continued under another id, in report order: a copy on another machine or in another * tool. `to` holds the turns of `from` and the ones that step added, so a step that would resume `from` resumes `to`. diff --git a/src/workflow/workspace.ts b/src/workflow/workspace.ts index 8ed3423cf..3f9102acc 100644 --- a/src/workflow/workspace.ts +++ b/src/workflow/workspace.ts @@ -5,8 +5,8 @@ import { itemsAfter, itemsOf } from './fan-out.js'; import { localId, statesIn, stepDefinitions, type Scope } from './scope.js'; import { parseStepPath } from './step-path.js'; -import { passOf, type StepState } from './step-state.js'; -import type { DriverIdentity, RunState, WorkflowFailure, WorkflowStep, WORKSPACE_MODES } from './types.js'; +import { passOf, type Delivery, type StepState } from './step-state.js'; +import type { DriverIdentity, RunState, WorkflowDefinition, WorkflowFailure, WorkflowStep, WORKSPACE_MODES } from './types.js'; export type WorkspaceMode = typeof WORKSPACE_MODES[number]; @@ -82,6 +82,51 @@ export function resolveMode(step: WorkflowStep, ancestors: WorkflowStep[]): Work return 'read-only'; } +/** + * Why a run of `definition` cannot deliver what its steps declare, or undefined when it can. A step in a sub-workflow + * continues the copy of the step that runs it, so this reads the whole tree a run executes, which one file cannot. + * `base` is the checkout the run starts from: a pull request targets a branch, and a detached HEAD has none. + */ +export function deliveryProblem( + definition: WorkflowDefinition, workflows: Record, base: RunState['base'], +): WorkflowFailure | undefined { + let delivering: string | undefined; + const childOf = (step: WorkflowStep) => (typeof step.workflow === 'string' ? workflows[step.workflow] : step.workflow); + const writes = (steps: WorkflowStep[], ancestors: WorkflowStep[]): boolean => steps.some((step) => { + const child = childOf(step); + if (child) return writes(child.steps, [...ancestors, step]); + return (step.prompt !== undefined || step.promptFile !== undefined || step.workspace === 'new' || step.workspace === 'continue') + && resolveMode(step, ancestors) !== 'read-only'; + }); + const visit = (steps: WorkflowStep[], ancestors: WorkflowStep[], prefix: string): WorkflowFailure | undefined => { + for (const step of steps) { + const path = `${prefix}${step.id}`; + const child = childOf(step); + if (step.deliver === 'pr') { + delivering ??= path; + if (step.workflow !== undefined && step.workspace !== 'new' && !step.forEach) { + return { code: 'bad_delivery', message: `Step "${path}" delivers its workflow, so it needs workspace: new to own the branch it delivers.` }; + } + if (child && !writes(child.steps, [...ancestors, step])) { + return { code: 'bad_delivery', message: `Step "${path}" has deliver: pr, but no step of the workflow it runs writes a branch, so it would deliver nothing. Set workspace: new on "${path}" so its prompt steps write, or give a step in that workflow workspace: new or continue.` }; + } + if (step.workflow === undefined && resolveMode(step, ancestors) === 'read-only') { + return { code: 'bad_delivery', message: `Step "${path}" has deliver: pr but runs read-only. Set workspace: new on it, or on the workflow step that runs it.` }; + } + } + const problem = child && visit(child.steps, [...ancestors, step], `${path}.`); + if (problem) return problem; + } + return undefined; + }; + const problem = visit(definition.steps, [], ''); + if (problem || delivering === undefined || !base || base.ref !== undefined) return problem; + return { + code: 'no_base_branch', + message: `Step "${delivering}" delivers a pull request, but the run would start from a detached HEAD at ${base.commit}, so no branch is there for the pull request to target. Check out a named branch, then start the run again.`, + }; +} + /** How step instance `path` runs, or undefined when the run's definitions have no such step. */ export function stepMode(run: StepTree, path: string): WorkspaceMode | undefined { const definition = stepDefinitions(run)(path); @@ -118,6 +163,69 @@ function changesOf(state: StepState | undefined): StepChanges | undefined { return state.changes; } +/** + * What a step that just succeeded delivers, or undefined when its branch holds nothing beyond the run base. A leaf + * delivers the branch it reported. A workflow step delivers its own copy once, never per step inside it; one without a + * copy of its own, such as a `for_each` item that runs a workflow, delivers the branch its latest step wrote. + */ +export function deliverableOf(run: RunState, state: StepState, step: WorkflowStep, changes?: StepChanges): Omit | undefined { + let branch = changes?.branch; + let head = changes?.to; + if (step.workflow !== undefined) { + branch = workflowBranch(run, state); + head = branch === undefined ? undefined : run.copies?.[branch]?.commit; + } + return branch === undefined || head === undefined || head === run.base?.commit ? undefined : { branch, head }; +} + +/** The branch workflow step `state` delivers: its own copy, else the branch its latest step wrote. */ +function workflowBranch(run: RunState, state: StepState): string | undefined { + const own = branchName(run.id, state.id, state.pass ?? 1); + if (run.copies?.[own]?.commit !== undefined) return own; + const latest = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status !== 'cancelled' && changesOf(item)) + .sort((left, right) => ('settledAt' in right ? right.settledAt : 0) - ('settledAt' in left ? left.settledAt : 0))[0]; + return changesOf(latest)?.branch; +} + +/** + * Work a step inside workflow step `state` wrote beyond the run base, when `state` delivers nothing of its own: no + * pull request would hold it. Undefined when there is none. + */ +export function workLeftUndelivered(run: RunState, state: StepState): { branch: string; head: string; child: { step: string; branch: string } } | undefined { + const branch = workflowBranch(run, state); + const base = run.base?.commit; + if (branch === undefined || base === undefined) return undefined; + const child = childWorkOff(run, state, branch).find((item) => item.commit !== base); + return child && { branch, head: base, child }; +} + +/** + * The work steps inside `state` that succeeded left on other branches than `branch`, the one `state` delivers, which no + * step inside it delivers: a pull request of `branch` leaves out any of it that `branch` does not hold. + */ +export function childWorkOff(run: RunState, state: StepState, branch: string): Array<{ step: string; branch: string; commit: string }> { + const inside = run.steps.filter((item) => item.id.startsWith(`${state.id}.`) && item.status === 'succeeded'); + const delivered = new Set(inside.flatMap((item) => ('delivery' in item && item.delivery ? [item.delivery.branch] : []))); + return inside.flatMap((item) => { + const changes = changesOf(item); + return changes && changes.branch !== branch && !delivered.has(changes.branch) ? [{ step: item.id, branch: changes.branch, commit: changes.to }] : []; + }); +} + +/** + * The branch a pull request from `branch` goes into: the nearest branch it grew from that delivered a pull request, so + * the request holds only the work done since, else the run base branch. Undefined when the run started from no branch. + */ +export function deliveryTarget(run: Pick, branch: string): string | undefined { + const delivered = new Set((run.deliveries ?? []).map((receipt) => receipt.branch)); + const walked = new Set([branch]); + for (let from = run.copies?.[branch]?.from; from !== undefined && !walked.has(from); from = run.copies?.[from]?.from) { + if (delivered.has(from)) return from; + walked.add(from); + } + return run.base?.ref; +} + /** Direct steps of a workflow step, such as `work.write`, not its fan-out items. */ function workflowChildren(run: RunState, step: StepState): StepState[] { const prefix = `${step.id}.`; @@ -131,13 +239,26 @@ function workflowChildren(run: RunState, step: StepState): StepState[] { /** Branches a step merges, `after` items first, then its other needs. An omitted `needs` uses the parent step's needs. */ function mergeChanges(run: RunState, scope: Scope, state: StepState, definition: WorkflowStep): StepChanges[] { const ordered: Array<{ changes: StepChanges; settledAt: number }> = []; + const visited = new Set(); const push = (step: StepState | undefined): void => { - if (!step) return; + if (!step || visited.has(step.id)) return; + visited.add(step.id); const changes = changesOf(step); // A for_each step keeps each item's branch on that item, and a workflow step keeps each child's on that child. if (!changes) { for (const item of itemsOf(run, step)) push(item); for (const child of workflowChildren(run, step)) push(child); + // A skipped or read-only need wrote no branch, but its earlier dependencies still supply the copy's origin. + let id: string | undefined = step.id; + while (id) { + const defined = stepDefinitions(run)(id); + const prefix: string = id.includes('.') ? id.slice(0, id.lastIndexOf('.') + 1) : ''; + if (defined?.needs !== undefined) { + for (const need of defined.needs) push(run.steps.find((item) => item.id === `${prefix}${need}`)); + break; + } + id = prefix ? prefix.slice(0, -1) : undefined; + } return; } const settledAt = 'settledAt' in step ? step.settledAt : 0; diff --git a/src/workflow/worktree.ts b/src/workflow/worktree.ts index b58054841..21bfca397 100644 --- a/src/workflow/worktree.ts +++ b/src/workflow/worktree.ts @@ -35,18 +35,36 @@ function configRoot(relative: string): string { return first !== relative && first.startsWith('.') && first !== '.github' ? first : relative; } +/** Paths a delivered branch may not change beyond AGENT_PATHS: CI and hooks, which run with someone's credentials. */ +export const DELIVERY_ONLY_PATHS: readonly string[] = [ + '.github/workflows', '.github/actions', '.github/CODEOWNERS', '.gitlab-ci.yml', '.gitlab', '.coding-ci.yaml', '.cnb.yml', '.cnb', + '.circleci', '.buildkite', '.husky', 'Jenkinsfile', 'azure-pipelines.yml', 'bitbucket-pipelines.yml', '.pre-commit-config.yaml', +]; + +/** + * The files among `changed` that no pull request TeamAI delivers may change: AGENT_PATHS, DELIVERY_ONLY_PATHS, and + * AGENTS.md or CLAUDE.md at any depth. Case-insensitive, as a checkout on a case-insensitive file system reads them. + */ +export function protectedPaths(changed: readonly string[]): string[] { + const roots = [...AGENT_PATHS, ...DELIVERY_ONLY_PATHS].map((root) => root.toLowerCase()); + return changed.filter((file) => { + const lower = file.toLowerCase(); + return /(^|\/)(agents|claude)\.md$/.test(lower) || roots.some((root) => lower === root || lower.startsWith(`${root}/`)); + }); +} + export interface GitResult { code: number; stdout: string; stderr: string; } -export function git(cwd: string, args: string[], input?: string): Promise { - if (input === undefined) { +/** Runs git in `cwd`, writing `input` to its stdin, or stopping it once `timeoutMs` passed. */ +export function git(cwd: string, args: string[], options?: { input: string } | { timeoutMs: number; env?: NodeJS.ProcessEnv }): Promise { + if (options === undefined || !('input' in options)) { return new Promise((resolve) => { - execFile('git', args, { cwd, env: process.env, windowsHide: true }, (error, stdout, stderr) => { - const failed = error as NodeJS.ErrnoException & { status?: number } | null; - const code = typeof failed?.status === 'number' ? failed.status : failed ? 1 : 0; + execFile('git', args, { cwd, env: { ...process.env, ...options?.env }, windowsHide: true, timeout: options?.timeoutMs }, (error, stdout, stderr) => { + const code = typeof error?.code === 'number' ? error.code : error ? 1 : 0; resolve({ code, stdout: String(stdout), stderr: String(stderr) }); }); }); @@ -67,7 +85,7 @@ export function git(cwd: string, args: string[], input?: string): Promise { stderr += chunk; }); child.on('error', () => finish(1)); child.on('close', (code) => finish(code ?? 1)); - child.stdin.end(input); + child.stdin.end(options.input); }); } @@ -226,12 +244,14 @@ async function worktreeHooks(main: string, dir: string, hooks: string): Promise< /** * The commits among `commits` this checkout still lacks after fetching run `run`'s branches from origin. A fetch whose * refspec matches no branch exits 0, so each commit is checked again after it. `fetchError` is git's message when the - * fetch itself failed. + * fetch itself failed. `fetch` bounds the fetch and sets its environment. */ -export async function fetchMissingCommits(dir: string, { run, commits }: { run: string; commits: string[] }): Promise<{ missing: string[]; fetchError?: string }> { +export async function fetchMissingCommits( + dir: string, { run, commits }: { run: string; commits: string[] }, fetch?: { timeoutMs: number; env?: NodeJS.ProcessEnv }, +): Promise<{ missing: string[]; fetchError?: string }> { const lacking = await lackedCommits(dir, commits); if (lacking.length === 0) return { missing: [] }; - const fetched = await git(dir, ['fetch', '--quiet', '--no-tags', 'origin', `+refs/heads/teamai/${run}/*:refs/remotes/origin/teamai/${run}/*`]); + const fetched = await git(dir, ['fetch', '--quiet', '--no-tags', 'origin', `+refs/heads/teamai/${run}/*:refs/remotes/origin/teamai/${run}/*`], fetch); const missing = await lackedCommits(dir, lacking); if (fetched.code === 0 || missing.length === 0) return { missing }; return { missing, fetchError: (fetched.stderr || fetched.stdout).trim() || 'git fetch failed' }; @@ -487,7 +507,7 @@ async function restoreConflictStages(dir: string, conflicts: string, paths: stri // `git rm --cached` leaves stage 0 in place beside the restored stages, and then `git add` cannot clear them. const removal = paths.map((relative) => `0 ${'0'.repeat(40)}\t${relative}`); const payload = [...removal, ...lines]; - if (payload.length > 0) await git(dir, ['update-index', '--index-info'], `${payload.join('\n')}\n`); + if (payload.length > 0) await git(dir, ['update-index', '--index-info'], { input: `${payload.join('\n')}\n` }); } async function fileManifest(root: string): Promise> {