From 42edce9e2471848fb25ece55c568697415e248d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 22:12:13 -0400 Subject: [PATCH 1/3] fix(daemon): forward daemon instance id on WebSocket task claims The WS claim path rebuilt the HTTP claim request from the connection identity but dropped X-Daemon-Instance-ID. Graph claims bind an attempt to the daemon process identity and refuse a claim without it, so every Graph task a daemon claimed over WebSocket was rejected and requeued immediately, in a loop, and the node never started. Reproduced on a local build of main (ad25feb0c) with a real Codex runtime: 684 claim/requeue cycles in about a minute, the agent never launched and the node stayed queued. With this change the same stuck task is claimed at once and the node runs. Capture the header at connect, forward it when rebuilding the claim request, and cover the forwarded headers with a regression test. Co-Authored-By: Claude Opus 5.5 --- server/internal/daemonws/hub.go | 4 +++ server/internal/handler/daemon_rpc.go | 31 ++++++++++++++----- server/internal/handler/daemon_rpc_test.go | 35 ++++++++++++++++++++++ server/internal/handler/daemon_ws.go | 1 + 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/server/internal/daemonws/hub.go b/server/internal/daemonws/hub.go index 3e717b722419..86e2c539581d 100644 --- a/server/internal/daemonws/hub.go +++ b/server/internal/daemonws/hub.go @@ -33,6 +33,10 @@ type ClientIdentity struct { // Capabilities is the raw X-Client-Capabilities header captured at connect, // so RPC handlers can honor the same capability gating as the HTTP path. Capabilities string + // InstanceID is the X-Daemon-Instance-ID header captured at connect. Graph + // claims bind an attempt to this process identity, so the WS claim path must + // forward it exactly as the HTTP path does. + InstanceID string } // AuthorizedWorkspaceIDs returns the connection's workspace scope in stable diff --git a/server/internal/handler/daemon_rpc.go b/server/internal/handler/daemon_rpc.go index 9a88332ba528..19f8465eb046 100644 --- a/server/internal/handler/daemon_rpc.go +++ b/server/internal/handler/daemon_rpc.go @@ -65,10 +65,29 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde if identity.DaemonID != "" { reqCtx = middleware.WithDaemonContext(reqCtx, identity.PrimaryWorkspaceID(), identity.DaemonID) } - req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body)) + req, err := newRPCClaimRequest(reqCtx, identity, body) if err != nil { return http.StatusInternalServerError, nil, err } + + rec := &rpcResponseCapture{} + h.ClaimTasksByRuntime(rec, req) + status := rec.status + if status == 0 { + status = http.StatusOK + } + return status, json.RawMessage(rec.body.Bytes()), nil +} + +// newRPCClaimRequest rebuilds the HTTP claim request a WS connection stands in +// for. Every header the claim path reads must be forwarded from the identity +// captured at connect; Graph claims in particular bind an attempt to +// X-Daemon-Instance-ID and refuse a claim without it. +func newRPCClaimRequest(ctx context.Context, identity daemonws.ClientIdentity, body json.RawMessage) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body)) + if err != nil { + return nil, err + } req.Header.Set("Content-Type", "application/json") if identity.UserID != "" { req.Header.Set("X-User-ID", identity.UserID) @@ -79,12 +98,8 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde if identity.ClientVersion != "" { req.Header.Set("X-Client-Version", identity.ClientVersion) } - - rec := &rpcResponseCapture{} - h.ClaimTasksByRuntime(rec, req) - status := rec.status - if status == 0 { - status = http.StatusOK + if identity.InstanceID != "" { + req.Header.Set("X-Daemon-Instance-ID", identity.InstanceID) } - return status, json.RawMessage(rec.body.Bytes()), nil + return req, nil } diff --git a/server/internal/handler/daemon_rpc_test.go b/server/internal/handler/daemon_rpc_test.go index 3d92a37e65a5..d248b6017dd9 100644 --- a/server/internal/handler/daemon_rpc_test.go +++ b/server/internal/handler/daemon_rpc_test.go @@ -72,3 +72,38 @@ func TestDaemonRPCHandler_UnknownMethod(t *testing.T) { t.Fatalf("status=%d err=%v, want 404 + error", status, err) } } + +// TestNewRPCClaimRequestForwardsConnectionIdentity pins that the synthetic claim +// request carries every header the HTTP claim path reads. A Graph attempt binds +// to the daemon process identity; without X-Daemon-Instance-ID the claim is +// refused and the task is requeued in a loop. +func TestNewRPCClaimRequestForwardsConnectionIdentity(t *testing.T) { + identity := daemonws.ClientIdentity{ + UserID: "user-1", + Capabilities: "graph-execution-v1,graph-execution-v2", + ClientVersion: "1.2.3", + InstanceID: "6a1f6f0c-8f0b-4a37-8a0e-0a1d3f1d2b11", + } + req, err := newRPCClaimRequest(context.Background(), identity, json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + for header, want := range map[string]string{ + "X-User-ID": identity.UserID, + "X-Client-Capabilities": identity.Capabilities, + "X-Client-Version": identity.ClientVersion, + "X-Daemon-Instance-ID": identity.InstanceID, + } { + if got := req.Header.Get(header); got != want { + t.Errorf("%s = %q, want %q", header, got, want) + } + } + + bare, err := newRPCClaimRequest(context.Background(), daemonws.ClientIdentity{}, nil) + if err != nil { + t.Fatal(err) + } + if got := bare.Header.Get("X-Daemon-Instance-ID"); got != "" { + t.Errorf("empty identity must not invent an instance id, got %q", got) + } +} diff --git a/server/internal/handler/daemon_ws.go b/server/internal/handler/daemon_ws.go index 89d78f5f3195..ee6243bb3627 100644 --- a/server/internal/handler/daemon_ws.go +++ b/server/internal/handler/daemon_ws.go @@ -54,6 +54,7 @@ func (h *Handler) DaemonWebSocket(w http.ResponseWriter, r *http.Request) { RuntimeIDs: runtimeIDs, ClientVersion: r.Header.Get("X-Client-Version"), Capabilities: r.Header.Get("X-Client-Capabilities"), + InstanceID: r.Header.Get("X-Daemon-Instance-ID"), }) } From 7afd6a62371b062a7d3bc0e299115eeeca0a3b51 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 22:12:13 -0400 Subject: [PATCH 2/3] fix(graph): carry physical stop evidence on cancelled, idle-killed and timed-out attempts When the server cancels a task, executeAndDrain returns from its drainCtx.Done branch immediately with a bare cancelled result, before the backend has finished tearing down its process tree. The backend computes ExecutionStopped only after that, so the result reached the daemon without stop evidence. A Graph attempt is settled only from a stop receipt, so the daemon withheld it (execution_unknown) and the engine, by design, kept the run blocked: a run cancelled mid-flight stayed in cancel_requested indefinitely. The idle-watchdog and drain-timeout branches had the same gap. For Graph attempts (DisableWholeRunRetry), wait up to 30s for the backend's terminal result in those branches and carry its ExecutionStopped. Ordinary tasks never read the evidence and are not delayed. Reproduced on a local build of main (ad25feb0c) with a real Codex runtime: a run cancelled while its node was executing was still cancel_requested after 80s. With this change it reaches cancelled in about 9s. A run that was already stuck before the fix is not healed by it and needs a manual cancel/archive. Co-Authored-By: Claude Opus 5.5 --- server/internal/daemon/daemon.go | 37 +++++++++++-- server/internal/daemon/daemon_test.go | 79 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/server/internal/daemon/daemon.go b/server/internal/daemon/daemon.go index acd959d3ead4..24f2b3752c36 100644 --- a/server/internal/daemon/daemon.go +++ b/server/internal/daemon/daemon.go @@ -8031,14 +8031,32 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro // hand back (and let runTask fail-and-broadcast) a still-flushing // transcript either. waitForDrain() + // A Graph attempt is settled only from a stop receipt, and the backend + // computes its physical stop proof after it has torn the process tree + // down — which is after this cancellation. Collect it (bounded) so a + // cancelled, idle-killed or timed-out attempt still carries it; otherwise + // the engine sees an unknown stop state and blocks the run indefinitely. + // Ordinary tasks never read it and must not be delayed. + var stopEvidence *bool + if opts.DisableWholeRunRetry { + select { + case late, ok := <-session.Result: + if ok { + stopEvidence = late.ExecutionStopped + } + case <-time.After(graphStopEvidenceWait): + taskLog.Warn("backend did not report physical stop evidence after cancellation") + } + } // Idle watchdog cancels via agentCancel(), which propagates here as // context.Canceled. Check this BEFORE the generic cancelled/timeout // classifiers so a watchdog-induced stop isn't misreported as // "task cancelled by server". if idleWatchdogFired.Load() { return agent.Result{ - Status: "idle_watchdog", - Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())), + Status: "idle_watchdog", + ExecutionStopped: stopEvidence, + Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())), }, toolCount.Load(), nil } // Distinguish external cancellation (e.g. server-initiated cancel @@ -8048,13 +8066,15 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro // drain deadline expiring on its own. if errors.Is(drainCtx.Err(), context.Canceled) { return agent.Result{ - Status: "cancelled", - Error: "task cancelled by upstream context (server cancel or daemon shutdown)", + Status: "cancelled", + ExecutionStopped: stopEvidence, + Error: "task cancelled by upstream context (server cancel or daemon shutdown)", }, toolCount.Load(), nil } return agent.Result{ - Status: "timeout", - Error: "agent did not produce result within drain timeout", + Status: "timeout", + ExecutionStopped: stopEvidence, + Error: "agent did not produce result within drain timeout", }, toolCount.Load(), nil } } @@ -8706,3 +8726,8 @@ func defaultArgsForProvider(cfg Config, provider string) []string { } return append([]string(nil), args...) } + +// graphStopEvidenceWait bounds how long a cancelled Graph attempt waits for the +// backend to finish tearing down its process tree and report stop evidence. It +// covers the Windows job-object and Unix process-group confirmation windows. +const graphStopEvidenceWait = 30 * time.Second diff --git a/server/internal/daemon/daemon_test.go b/server/internal/daemon/daemon_test.go index 8c2affff8790..4b857c3cdeae 100644 --- a/server/internal/daemon/daemon_test.go +++ b/server/internal/daemon/daemon_test.go @@ -5602,3 +5602,82 @@ func TestHermesProfileChainCoversLaunchPrefix(t *testing.T) { t.Errorf("custom = %v, want only the selector removed", strippedCustom) } } + +// lateStopEvidenceBackend models a backend that keeps tearing its process tree +// down after the daemon cancels the run: the terminal Result, carrying the +// physical stop proof, only arrives once the context is cancelled and a short +// reap delay has passed. +type lateStopEvidenceBackend struct { + stopped bool + delay time.Duration +} + +func (b lateStopEvidenceBackend) Execute(ctx context.Context, _ string, _ agent.ExecOptions) (*agent.Session, error) { + msgCh := make(chan agent.Message) + resCh := make(chan agent.Result, 1) + go func() { + <-ctx.Done() + time.Sleep(b.delay) + close(msgCh) + stopped := b.stopped + resCh <- agent.Result{Status: "aborted", ExecutionStopped: &stopped} + }() + return &agent.Session{Messages: msgCh, Result: resCh}, nil +} + +// A cancelled Graph attempt can only be settled from a stop receipt, so the +// cancel path must wait for the backend's physical stop proof instead of +// returning a result that lacks it (which left the run blocked forever). +func TestExecuteAndDrain_GraphCancelCarriesStopEvidence(t *testing.T) { + t.Parallel() + + for name, stopped := range map[string]bool{"confirmed": true, "unconfirmed": false} { + t.Run(name, func(t *testing.T) { + t.Parallel() + d := newTestDaemon(t) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: stopped, delay: 150 * time.Millisecond}, "p", + agent.ExecOptions{DisableWholeRunRetry: true}, slog.Default(), "t-graph-cancel", "", new(atomic.Int32)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Status != "cancelled" { + t.Fatalf("status = %q, want cancelled", result.Status) + } + if result.ExecutionStopped == nil { + t.Fatal("cancelled Graph result dropped the backend's stop evidence") + } + if *result.ExecutionStopped != stopped { + t.Fatalf("ExecutionStopped = %v, want %v", *result.ExecutionStopped, stopped) + } + }) + } +} + +// Ordinary tasks never read stop evidence, so a cancel must not be delayed +// waiting for it. +func TestExecuteAndDrain_NonGraphCancelDoesNotWaitForBackend(t *testing.T) { + t.Parallel() + + d := newTestDaemon(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: true, delay: 3 * time.Second}, "p", + agent.ExecOptions{}, slog.Default(), "t-plain-cancel", "", new(atomic.Int32)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Status != "cancelled" || result.ExecutionStopped != nil { + t.Fatalf("result = %+v, want cancelled without stop evidence", result) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("non-Graph cancel took %v; it must not wait for the backend", elapsed) + } +} From c9e582714163de5ebdf5180805499364fd085ece Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 22:12:14 -0400 Subject: [PATCH 3/3] fix(graph): reject non-UUID artifact ids at submission and tell agents what to use A produce_commit agent declared its commit SHA as the git_commit artifact id. The prompt never said what the id must be and ValidateResult only checked that it was non-empty, so the submission was accepted. The id is stored in a uuid column, so reconciliation then failed with "invalid_id: invalid Graph resource UUID" every 3 seconds, permanently, and the run could not advance; only cancelling it recovered. Validate the artifact id as a UUID when the result is submitted, so the agent gets a correctable error, and state the requirement in the node prompt and in the built-in working-on-issues skill. Co-Authored-By: Claude Opus 5.5 --- server/internal/daemon/graph.go | 2 +- server/internal/graph/result.go | 6 +++++ server/internal/graph/result_test.go | 25 +++++++++++++++++++ .../multica-working-on-issues/SKILL.md | 3 ++- .../working-on-issues-source-map.md | 3 ++- 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/server/internal/daemon/graph.go b/server/internal/daemon/graph.go index 0fadfde3da2c..49dd4a74c23f 100644 --- a/server/internal/daemon/graph.go +++ b/server/internal/daemon/graph.go @@ -140,7 +140,7 @@ func buildGraphPrompt(task Task) string { fmt.Fprintf(&b, "The repository has already been checked out beneath the task working directory at input commit %s. Work in that checkout; do not replace the input ref.\n", task.Graph.NodeSpec.Repository.CommitSHA) } if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionProduceCommit { - fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA) + fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Give the artifact a freshly generated UUID as its id (not the commit SHA). Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA) } if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionVerifyCommit { b.WriteString("Keep HEAD at the input candidate and leave the checkout clean. Return tested_commit_sha equal to that input commit.\n") diff --git a/server/internal/graph/result.go b/server/internal/graph/result.go index e12bde71afe6..c7a50eedf0a7 100644 --- a/server/internal/graph/result.go +++ b/server/internal/graph/result.go @@ -12,6 +12,7 @@ const ProtocolVersion = 1 var hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`) var commitPattern = regexp.MustCompile(`^([a-f0-9]{40}|[a-f0-9]{64})$`) +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) type AttemptIdentity struct { RunID string `json:"run_id"` @@ -240,6 +241,11 @@ func ValidateResult(node Node, result Result) error { if a.ID == "" || seen[a.ID] || a.ProducerTaskID == "" { return invalid("artifacts", "requires unique artifact IDs and producer identity") } + // Artifact IDs are stored in a uuid column; rejecting anything else here lets + // the agent correct it instead of wedging reconciliation after acceptance. + if !uuidPattern.MatchString(a.ID) { + return invalid("artifacts", "artifact id must be a UUID (generate a new one; it is not the commit SHA)") + } seen[a.ID] = true switch a.Type { case ArtifactGitCommit: diff --git a/server/internal/graph/result_test.go b/server/internal/graph/result_test.go index e27e17c53f59..d6a5d417d8a1 100644 --- a/server/internal/graph/result_test.go +++ b/server/internal/graph/result_test.go @@ -110,3 +110,28 @@ func TestCanonicalJSONPreservesLargeNumbersAndNormalizesSpelling(t *testing.T) { t.Fatalf("lost numeric identity: %s", a) } } + +// Artifact identities are persisted in a uuid column. An ID that is not a UUID +// used to pass submission and then fail reconciliation forever, wedging the +// run in an unrecoverable loop; it must be rejected when the result is +// submitted so the agent can correct it. +func TestResultRejectsNonUUIDArtifactIDAtSubmission(t *testing.T) { + n := testDefinition().Nodes[1] + n.ExecutionMode = ExecutionProduceCommit + id := testIdentity() + sha := "3cbd6fad2097feff64ecd57deaa2d6df4f275d21" + build := func(artifactID string) Result { + return Result{Identity: id, Outcome: OutcomePass, Output: json.RawMessage(`{"summary":"done"}`), Artifacts: []Artifact{{ + ID: artifactID, Type: ArtifactGitCommit, RepositoryID: "repo", BaseSHA: "d41e3e2254ced0bd5c6d4c0b5fb82dcb772ff442", + CommitSHA: sha, ManagedRef: "refs/heads/multica/graphs/run/node/1", ProducerTaskID: id.TaskID, + }}} + } + if err := ValidateResult(n, build("7b0f8f0e-8c53-4a4e-9a58-0d3c1c2f6a11")); err != nil { + t.Fatalf("uuid artifact id rejected: %v", err) + } + for _, bad := range []string{sha, "artifact-1", "not a uuid"} { + if err := ValidateResult(n, build(bad)); err == nil { + t.Fatalf("artifact id %q accepted at submission", bad) + } + } +} diff --git a/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md b/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md index 3f8ca5f35aa7..3d4ea9c9115a 100644 --- a/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md +++ b/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md @@ -41,7 +41,8 @@ is a nonempty diagnostics object, for example the repository checkout. An unchanged result reuses a stable idempotency key. For `produce_commit`, commit locally and declare the exact `git_commit` artifact -described in the execution manifest and prompt; the daemon verifies and pushes +described in the execution manifest and prompt, with a freshly generated UUID as +its `id` (never the commit SHA); the daemon verifies and pushes the managed ref after execution stops. For `verify_commit`, keep HEAD at the frozen candidate, leave the checkout clean and return its `tested_commit_sha`. Do not replace structured submission with a comment, PR, exit code or a manual diff --git a/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md b/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md index 42446784f2e1..112151a622ed 100644 --- a/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md +++ b/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md @@ -18,7 +18,8 @@ - `server/internal/daemon/graph.go`: `buildGraphPrompt`, `finalizeGraphGit` and `finalizeGraphTask` define structured delivery and stopped receipts. - `server/internal/graph/result.go`: `ValidateResult` enforces execution-mode - artifacts, output schema, diagnostics and the tested candidate SHA. + artifacts (artifact ids must be UUIDs), output schema, diagnostics and the + tested candidate SHA. Evidence layer for `SKILL.md`. Every contract the skill states is traced to a current `file:line` here. Lines were re-derived against `feat/builtin-skills`