Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 31 additions & 6 deletions server/internal/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -8031,14 +8031,32 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro
// hand back (and let runTask fail-and-broadcast) a still-flushing
// transcript either.
waitForDrain()
// A Graph attempt is settled only from a stop receipt, and the backend
// computes its physical stop proof after it has torn the process tree
// down — which is after this cancellation. Collect it (bounded) so a
// cancelled, idle-killed or timed-out attempt still carries it; otherwise
// the engine sees an unknown stop state and blocks the run indefinitely.
// Ordinary tasks never read it and must not be delayed.
var stopEvidence *bool
if opts.DisableWholeRunRetry {
select {
case late, ok := <-session.Result:
if ok {
stopEvidence = late.ExecutionStopped
}
case <-time.After(graphStopEvidenceWait):
taskLog.Warn("backend did not report physical stop evidence after cancellation")
}
}
// Idle watchdog cancels via agentCancel(), which propagates here as
// context.Canceled. Check this BEFORE the generic cancelled/timeout
// classifiers so a watchdog-induced stop isn't misreported as
// "task cancelled by server".
if idleWatchdogFired.Load() {
return agent.Result{
Status: "idle_watchdog",
Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())),
Status: "idle_watchdog",
ExecutionStopped: stopEvidence,
Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())),
}, toolCount.Load(), nil
}
// Distinguish external cancellation (e.g. server-initiated cancel
Expand All @@ -8048,13 +8066,15 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro
// drain deadline expiring on its own.
if errors.Is(drainCtx.Err(), context.Canceled) {
return agent.Result{
Status: "cancelled",
Error: "task cancelled by upstream context (server cancel or daemon shutdown)",
Status: "cancelled",
ExecutionStopped: stopEvidence,
Error: "task cancelled by upstream context (server cancel or daemon shutdown)",
}, toolCount.Load(), nil
}
return agent.Result{
Status: "timeout",
Error: "agent did not produce result within drain timeout",
Status: "timeout",
ExecutionStopped: stopEvidence,
Error: "agent did not produce result within drain timeout",
}, toolCount.Load(), nil
}
}
Expand Down Expand Up @@ -8706,3 +8726,8 @@ func defaultArgsForProvider(cfg Config, provider string) []string {
}
return append([]string(nil), args...)
}

// graphStopEvidenceWait bounds how long a cancelled Graph attempt waits for the
// backend to finish tearing down its process tree and report stop evidence. It
// covers the Windows job-object and Unix process-group confirmation windows.
const graphStopEvidenceWait = 30 * time.Second
79 changes: 79 additions & 0 deletions server/internal/daemon/daemon_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5602,3 +5602,82 @@ func TestHermesProfileChainCoversLaunchPrefix(t *testing.T) {
t.Errorf("custom = %v, want only the selector removed", strippedCustom)
}
}

// lateStopEvidenceBackend models a backend that keeps tearing its process tree
// down after the daemon cancels the run: the terminal Result, carrying the
// physical stop proof, only arrives once the context is cancelled and a short
// reap delay has passed.
type lateStopEvidenceBackend struct {
stopped bool
delay time.Duration
}

func (b lateStopEvidenceBackend) Execute(ctx context.Context, _ string, _ agent.ExecOptions) (*agent.Session, error) {
msgCh := make(chan agent.Message)
resCh := make(chan agent.Result, 1)
go func() {
<-ctx.Done()
time.Sleep(b.delay)
close(msgCh)
stopped := b.stopped
resCh <- agent.Result{Status: "aborted", ExecutionStopped: &stopped}
}()
return &agent.Session{Messages: msgCh, Result: resCh}, nil
}

// A cancelled Graph attempt can only be settled from a stop receipt, so the
// cancel path must wait for the backend's physical stop proof instead of
// returning a result that lacks it (which left the run blocked forever).
func TestExecuteAndDrain_GraphCancelCarriesStopEvidence(t *testing.T) {
t.Parallel()

for name, stopped := range map[string]bool{"confirmed": true, "unconfirmed": false} {
t.Run(name, func(t *testing.T) {
t.Parallel()
d := newTestDaemon(t)
ctx, cancel := context.WithCancel(context.Background())
go func() {
time.Sleep(20 * time.Millisecond)
cancel()
}()

result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: stopped, delay: 150 * time.Millisecond}, "p",
agent.ExecOptions{DisableWholeRunRetry: true}, slog.Default(), "t-graph-cancel", "", new(atomic.Int32))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Status != "cancelled" {
t.Fatalf("status = %q, want cancelled", result.Status)
}
if result.ExecutionStopped == nil {
t.Fatal("cancelled Graph result dropped the backend's stop evidence")
}
if *result.ExecutionStopped != stopped {
t.Fatalf("ExecutionStopped = %v, want %v", *result.ExecutionStopped, stopped)
}
})
}
}

// Ordinary tasks never read stop evidence, so a cancel must not be delayed
// waiting for it.
func TestExecuteAndDrain_NonGraphCancelDoesNotWaitForBackend(t *testing.T) {
t.Parallel()

d := newTestDaemon(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()

start := time.Now()
result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: true, delay: 3 * time.Second}, "p",
agent.ExecOptions{}, slog.Default(), "t-plain-cancel", "", new(atomic.Int32))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Status != "cancelled" || result.ExecutionStopped != nil {
t.Fatalf("result = %+v, want cancelled without stop evidence", result)
}
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("non-Graph cancel took %v; it must not wait for the backend", elapsed)
}
}
2 changes: 1 addition & 1 deletion server/internal/daemon/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func buildGraphPrompt(task Task) string {
fmt.Fprintf(&b, "The repository has already been checked out beneath the task working directory at input commit %s. Work in that checkout; do not replace the input ref.\n", task.Graph.NodeSpec.Repository.CommitSHA)
}
if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionProduceCommit {
fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA)
fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Give the artifact a freshly generated UUID as its id (not the commit SHA). Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA)
}
if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionVerifyCommit {
b.WriteString("Keep HEAD at the input candidate and leave the checkout clean. Return tested_commit_sha equal to that input commit.\n")
Expand Down
4 changes: 4 additions & 0 deletions server/internal/daemonws/hub.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ type ClientIdentity struct {
// Capabilities is the raw X-Client-Capabilities header captured at connect,
// so RPC handlers can honor the same capability gating as the HTTP path.
Capabilities string
// InstanceID is the X-Daemon-Instance-ID header captured at connect. Graph
// claims bind an attempt to this process identity, so the WS claim path must
// forward it exactly as the HTTP path does.
InstanceID string
}

// AuthorizedWorkspaceIDs returns the connection's workspace scope in stable
Expand Down
6 changes: 6 additions & 0 deletions server/internal/graph/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const ProtocolVersion = 1

var hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`)
var commitPattern = regexp.MustCompile(`^([a-f0-9]{40}|[a-f0-9]{64})$`)
var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)

type AttemptIdentity struct {
RunID string `json:"run_id"`
Expand Down Expand Up @@ -240,6 +241,11 @@ func ValidateResult(node Node, result Result) error {
if a.ID == "" || seen[a.ID] || a.ProducerTaskID == "" {
return invalid("artifacts", "requires unique artifact IDs and producer identity")
}
// Artifact IDs are stored in a uuid column; rejecting anything else here lets
// the agent correct it instead of wedging reconciliation after acceptance.
if !uuidPattern.MatchString(a.ID) {
return invalid("artifacts", "artifact id must be a UUID (generate a new one; it is not the commit SHA)")
}
seen[a.ID] = true
switch a.Type {
case ArtifactGitCommit:
Expand Down
25 changes: 25 additions & 0 deletions server/internal/graph/result_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,3 +110,28 @@ func TestCanonicalJSONPreservesLargeNumbersAndNormalizesSpelling(t *testing.T) {
t.Fatalf("lost numeric identity: %s", a)
}
}

// Artifact identities are persisted in a uuid column. An ID that is not a UUID
// used to pass submission and then fail reconciliation forever, wedging the
// run in an unrecoverable loop; it must be rejected when the result is
// submitted so the agent can correct it.
func TestResultRejectsNonUUIDArtifactIDAtSubmission(t *testing.T) {
n := testDefinition().Nodes[1]
n.ExecutionMode = ExecutionProduceCommit
id := testIdentity()
sha := "3cbd6fad2097feff64ecd57deaa2d6df4f275d21"
build := func(artifactID string) Result {
return Result{Identity: id, Outcome: OutcomePass, Output: json.RawMessage(`{"summary":"done"}`), Artifacts: []Artifact{{
ID: artifactID, Type: ArtifactGitCommit, RepositoryID: "repo", BaseSHA: "d41e3e2254ced0bd5c6d4c0b5fb82dcb772ff442",
CommitSHA: sha, ManagedRef: "refs/heads/multica/graphs/run/node/1", ProducerTaskID: id.TaskID,
}}}
}
if err := ValidateResult(n, build("7b0f8f0e-8c53-4a4e-9a58-0d3c1c2f6a11")); err != nil {
t.Fatalf("uuid artifact id rejected: %v", err)
}
for _, bad := range []string{sha, "artifact-1", "not a uuid"} {
if err := ValidateResult(n, build(bad)); err == nil {
t.Fatalf("artifact id %q accepted at submission", bad)
}
}
}
31 changes: 23 additions & 8 deletions server/internal/handler/daemon_rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,29 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde
if identity.DaemonID != "" {
reqCtx = middleware.WithDaemonContext(reqCtx, identity.PrimaryWorkspaceID(), identity.DaemonID)
}
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body))
req, err := newRPCClaimRequest(reqCtx, identity, body)
if err != nil {
return http.StatusInternalServerError, nil, err
}

rec := &rpcResponseCapture{}
h.ClaimTasksByRuntime(rec, req)
status := rec.status
if status == 0 {
status = http.StatusOK
}
return status, json.RawMessage(rec.body.Bytes()), nil
}

// newRPCClaimRequest rebuilds the HTTP claim request a WS connection stands in
// for. Every header the claim path reads must be forwarded from the identity
// captured at connect; Graph claims in particular bind an attempt to
// X-Daemon-Instance-ID and refuse a claim without it.
func newRPCClaimRequest(ctx context.Context, identity daemonws.ClientIdentity, body json.RawMessage) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if identity.UserID != "" {
req.Header.Set("X-User-ID", identity.UserID)
Expand All @@ -79,12 +98,8 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde
if identity.ClientVersion != "" {
req.Header.Set("X-Client-Version", identity.ClientVersion)
}

rec := &rpcResponseCapture{}
h.ClaimTasksByRuntime(rec, req)
status := rec.status
if status == 0 {
status = http.StatusOK
if identity.InstanceID != "" {
req.Header.Set("X-Daemon-Instance-ID", identity.InstanceID)
}
return status, json.RawMessage(rec.body.Bytes()), nil
return req, nil
}
35 changes: 35 additions & 0 deletions server/internal/handler/daemon_rpc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,38 @@ func TestDaemonRPCHandler_UnknownMethod(t *testing.T) {
t.Fatalf("status=%d err=%v, want 404 + error", status, err)
}
}

// TestNewRPCClaimRequestForwardsConnectionIdentity pins that the synthetic claim
// request carries every header the HTTP claim path reads. A Graph attempt binds
// to the daemon process identity; without X-Daemon-Instance-ID the claim is
// refused and the task is requeued in a loop.
func TestNewRPCClaimRequestForwardsConnectionIdentity(t *testing.T) {
identity := daemonws.ClientIdentity{
UserID: "user-1",
Capabilities: "graph-execution-v1,graph-execution-v2",
ClientVersion: "1.2.3",
InstanceID: "6a1f6f0c-8f0b-4a37-8a0e-0a1d3f1d2b11",
}
req, err := newRPCClaimRequest(context.Background(), identity, json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
for header, want := range map[string]string{
"X-User-ID": identity.UserID,
"X-Client-Capabilities": identity.Capabilities,
"X-Client-Version": identity.ClientVersion,
"X-Daemon-Instance-ID": identity.InstanceID,
} {
if got := req.Header.Get(header); got != want {
t.Errorf("%s = %q, want %q", header, got, want)
}
}

bare, err := newRPCClaimRequest(context.Background(), daemonws.ClientIdentity{}, nil)
if err != nil {
t.Fatal(err)
}
if got := bare.Header.Get("X-Daemon-Instance-ID"); got != "" {
t.Errorf("empty identity must not invent an instance id, got %q", got)
}
}
1 change: 1 addition & 0 deletions server/internal/handler/daemon_ws.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func (h *Handler) DaemonWebSocket(w http.ResponseWriter, r *http.Request) {
RuntimeIDs: runtimeIDs,
ClientVersion: r.Header.Get("X-Client-Version"),
Capabilities: r.Header.Get("X-Client-Capabilities"),
InstanceID: r.Header.Get("X-Daemon-Instance-ID"),
})
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ is a nonempty diagnostics object, for example
the repository checkout. An unchanged result reuses a stable idempotency key.

For `produce_commit`, commit locally and declare the exact `git_commit` artifact
described in the execution manifest and prompt; the daemon verifies and pushes
described in the execution manifest and prompt, with a freshly generated UUID as
its `id` (never the commit SHA); the daemon verifies and pushes
the managed ref after execution stops. For `verify_commit`, keep HEAD at the
frozen candidate, leave the checkout clean and return its `tested_commit_sha`.
Do not replace structured submission with a comment, PR, exit code or a manual
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
- `server/internal/daemon/graph.go`: `buildGraphPrompt`, `finalizeGraphGit` and
`finalizeGraphTask` define structured delivery and stopped receipts.
- `server/internal/graph/result.go`: `ValidateResult` enforces execution-mode
artifacts, output schema, diagnostics and the tested candidate SHA.
artifacts (artifact ids must be UUIDs), output schema, diagnostics and the
tested candidate SHA.

Evidence layer for `SKILL.md`. Every contract the skill states is traced to a
current `file:line` here. Lines were re-derived against `feat/builtin-skills`
Expand Down
Loading