diff --git a/server/internal/daemon/daemon.go b/server/internal/daemon/daemon.go index acd959d3ead4..24f2b3752c36 100644 --- a/server/internal/daemon/daemon.go +++ b/server/internal/daemon/daemon.go @@ -8031,14 +8031,32 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro // hand back (and let runTask fail-and-broadcast) a still-flushing // transcript either. waitForDrain() + // A Graph attempt is settled only from a stop receipt, and the backend + // computes its physical stop proof after it has torn the process tree + // down — which is after this cancellation. Collect it (bounded) so a + // cancelled, idle-killed or timed-out attempt still carries it; otherwise + // the engine sees an unknown stop state and blocks the run indefinitely. + // Ordinary tasks never read it and must not be delayed. + var stopEvidence *bool + if opts.DisableWholeRunRetry { + select { + case late, ok := <-session.Result: + if ok { + stopEvidence = late.ExecutionStopped + } + case <-time.After(graphStopEvidenceWait): + taskLog.Warn("backend did not report physical stop evidence after cancellation") + } + } // Idle watchdog cancels via agentCancel(), which propagates here as // context.Canceled. Check this BEFORE the generic cancelled/timeout // classifiers so a watchdog-induced stop isn't misreported as // "task cancelled by server". if idleWatchdogFired.Load() { return agent.Result{ - Status: "idle_watchdog", - Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())), + Status: "idle_watchdog", + ExecutionStopped: stopEvidence, + Error: idleWatchdogReason(time.Duration(idleWatchdogThreshold.Load())), }, toolCount.Load(), nil } // Distinguish external cancellation (e.g. server-initiated cancel @@ -8048,13 +8066,15 @@ func (d *Daemon) executeAndDrain(ctx context.Context, backend agent.Backend, pro // drain deadline expiring on its own. if errors.Is(drainCtx.Err(), context.Canceled) { return agent.Result{ - Status: "cancelled", - Error: "task cancelled by upstream context (server cancel or daemon shutdown)", + Status: "cancelled", + ExecutionStopped: stopEvidence, + Error: "task cancelled by upstream context (server cancel or daemon shutdown)", }, toolCount.Load(), nil } return agent.Result{ - Status: "timeout", - Error: "agent did not produce result within drain timeout", + Status: "timeout", + ExecutionStopped: stopEvidence, + Error: "agent did not produce result within drain timeout", }, toolCount.Load(), nil } } @@ -8706,3 +8726,8 @@ func defaultArgsForProvider(cfg Config, provider string) []string { } return append([]string(nil), args...) } + +// graphStopEvidenceWait bounds how long a cancelled Graph attempt waits for the +// backend to finish tearing down its process tree and report stop evidence. It +// covers the Windows job-object and Unix process-group confirmation windows. +const graphStopEvidenceWait = 30 * time.Second diff --git a/server/internal/daemon/daemon_test.go b/server/internal/daemon/daemon_test.go index 8c2affff8790..4b857c3cdeae 100644 --- a/server/internal/daemon/daemon_test.go +++ b/server/internal/daemon/daemon_test.go @@ -5602,3 +5602,82 @@ func TestHermesProfileChainCoversLaunchPrefix(t *testing.T) { t.Errorf("custom = %v, want only the selector removed", strippedCustom) } } + +// lateStopEvidenceBackend models a backend that keeps tearing its process tree +// down after the daemon cancels the run: the terminal Result, carrying the +// physical stop proof, only arrives once the context is cancelled and a short +// reap delay has passed. +type lateStopEvidenceBackend struct { + stopped bool + delay time.Duration +} + +func (b lateStopEvidenceBackend) Execute(ctx context.Context, _ string, _ agent.ExecOptions) (*agent.Session, error) { + msgCh := make(chan agent.Message) + resCh := make(chan agent.Result, 1) + go func() { + <-ctx.Done() + time.Sleep(b.delay) + close(msgCh) + stopped := b.stopped + resCh <- agent.Result{Status: "aborted", ExecutionStopped: &stopped} + }() + return &agent.Session{Messages: msgCh, Result: resCh}, nil +} + +// A cancelled Graph attempt can only be settled from a stop receipt, so the +// cancel path must wait for the backend's physical stop proof instead of +// returning a result that lacks it (which left the run blocked forever). +func TestExecuteAndDrain_GraphCancelCarriesStopEvidence(t *testing.T) { + t.Parallel() + + for name, stopped := range map[string]bool{"confirmed": true, "unconfirmed": false} { + t.Run(name, func(t *testing.T) { + t.Parallel() + d := newTestDaemon(t) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + + result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: stopped, delay: 150 * time.Millisecond}, "p", + agent.ExecOptions{DisableWholeRunRetry: true}, slog.Default(), "t-graph-cancel", "", new(atomic.Int32)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Status != "cancelled" { + t.Fatalf("status = %q, want cancelled", result.Status) + } + if result.ExecutionStopped == nil { + t.Fatal("cancelled Graph result dropped the backend's stop evidence") + } + if *result.ExecutionStopped != stopped { + t.Fatalf("ExecutionStopped = %v, want %v", *result.ExecutionStopped, stopped) + } + }) + } +} + +// Ordinary tasks never read stop evidence, so a cancel must not be delayed +// waiting for it. +func TestExecuteAndDrain_NonGraphCancelDoesNotWaitForBackend(t *testing.T) { + t.Parallel() + + d := newTestDaemon(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + result, _, err := d.executeAndDrain(ctx, lateStopEvidenceBackend{stopped: true, delay: 3 * time.Second}, "p", + agent.ExecOptions{}, slog.Default(), "t-plain-cancel", "", new(atomic.Int32)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Status != "cancelled" || result.ExecutionStopped != nil { + t.Fatalf("result = %+v, want cancelled without stop evidence", result) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("non-Graph cancel took %v; it must not wait for the backend", elapsed) + } +} diff --git a/server/internal/daemon/graph.go b/server/internal/daemon/graph.go index 0fadfde3da2c..49dd4a74c23f 100644 --- a/server/internal/daemon/graph.go +++ b/server/internal/daemon/graph.go @@ -140,7 +140,7 @@ func buildGraphPrompt(task Task) string { fmt.Fprintf(&b, "The repository has already been checked out beneath the task working directory at input commit %s. Work in that checkout; do not replace the input ref.\n", task.Graph.NodeSpec.Repository.CommitSHA) } if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionProduceCommit { - fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA) + fmt.Fprintf(&b, "Commit your completed code explicitly and declare that exact SHA in a git_commit artifact. Give the artifact a freshly generated UUID as its id (not the commit SHA). Its managed_ref must be %s, repository_id %s, producer_task_id %s, and base_sha %s. The daemon publishes it after you exit; do not push the managed ref yourself.\n", graphManagedRef(task), task.Graph.NodeSpec.Repository.RepositoryID, task.ID, task.Graph.NodeSpec.Repository.CommitSHA) } if task.Graph.NodeSpec.Node.ExecutionMode == graph.ExecutionVerifyCommit { b.WriteString("Keep HEAD at the input candidate and leave the checkout clean. Return tested_commit_sha equal to that input commit.\n") diff --git a/server/internal/daemonws/hub.go b/server/internal/daemonws/hub.go index 3e717b722419..86e2c539581d 100644 --- a/server/internal/daemonws/hub.go +++ b/server/internal/daemonws/hub.go @@ -33,6 +33,10 @@ type ClientIdentity struct { // Capabilities is the raw X-Client-Capabilities header captured at connect, // so RPC handlers can honor the same capability gating as the HTTP path. Capabilities string + // InstanceID is the X-Daemon-Instance-ID header captured at connect. Graph + // claims bind an attempt to this process identity, so the WS claim path must + // forward it exactly as the HTTP path does. + InstanceID string } // AuthorizedWorkspaceIDs returns the connection's workspace scope in stable diff --git a/server/internal/graph/result.go b/server/internal/graph/result.go index e12bde71afe6..c7a50eedf0a7 100644 --- a/server/internal/graph/result.go +++ b/server/internal/graph/result.go @@ -12,6 +12,7 @@ const ProtocolVersion = 1 var hashPattern = regexp.MustCompile(`^[a-f0-9]{64}$`) var commitPattern = regexp.MustCompile(`^([a-f0-9]{40}|[a-f0-9]{64})$`) +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) type AttemptIdentity struct { RunID string `json:"run_id"` @@ -240,6 +241,11 @@ func ValidateResult(node Node, result Result) error { if a.ID == "" || seen[a.ID] || a.ProducerTaskID == "" { return invalid("artifacts", "requires unique artifact IDs and producer identity") } + // Artifact IDs are stored in a uuid column; rejecting anything else here lets + // the agent correct it instead of wedging reconciliation after acceptance. + if !uuidPattern.MatchString(a.ID) { + return invalid("artifacts", "artifact id must be a UUID (generate a new one; it is not the commit SHA)") + } seen[a.ID] = true switch a.Type { case ArtifactGitCommit: diff --git a/server/internal/graph/result_test.go b/server/internal/graph/result_test.go index e27e17c53f59..d6a5d417d8a1 100644 --- a/server/internal/graph/result_test.go +++ b/server/internal/graph/result_test.go @@ -110,3 +110,28 @@ func TestCanonicalJSONPreservesLargeNumbersAndNormalizesSpelling(t *testing.T) { t.Fatalf("lost numeric identity: %s", a) } } + +// Artifact identities are persisted in a uuid column. An ID that is not a UUID +// used to pass submission and then fail reconciliation forever, wedging the +// run in an unrecoverable loop; it must be rejected when the result is +// submitted so the agent can correct it. +func TestResultRejectsNonUUIDArtifactIDAtSubmission(t *testing.T) { + n := testDefinition().Nodes[1] + n.ExecutionMode = ExecutionProduceCommit + id := testIdentity() + sha := "3cbd6fad2097feff64ecd57deaa2d6df4f275d21" + build := func(artifactID string) Result { + return Result{Identity: id, Outcome: OutcomePass, Output: json.RawMessage(`{"summary":"done"}`), Artifacts: []Artifact{{ + ID: artifactID, Type: ArtifactGitCommit, RepositoryID: "repo", BaseSHA: "d41e3e2254ced0bd5c6d4c0b5fb82dcb772ff442", + CommitSHA: sha, ManagedRef: "refs/heads/multica/graphs/run/node/1", ProducerTaskID: id.TaskID, + }}} + } + if err := ValidateResult(n, build("7b0f8f0e-8c53-4a4e-9a58-0d3c1c2f6a11")); err != nil { + t.Fatalf("uuid artifact id rejected: %v", err) + } + for _, bad := range []string{sha, "artifact-1", "not a uuid"} { + if err := ValidateResult(n, build(bad)); err == nil { + t.Fatalf("artifact id %q accepted at submission", bad) + } + } +} diff --git a/server/internal/handler/daemon_rpc.go b/server/internal/handler/daemon_rpc.go index 9a88332ba528..19f8465eb046 100644 --- a/server/internal/handler/daemon_rpc.go +++ b/server/internal/handler/daemon_rpc.go @@ -65,10 +65,29 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde if identity.DaemonID != "" { reqCtx = middleware.WithDaemonContext(reqCtx, identity.PrimaryWorkspaceID(), identity.DaemonID) } - req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body)) + req, err := newRPCClaimRequest(reqCtx, identity, body) if err != nil { return http.StatusInternalServerError, nil, err } + + rec := &rpcResponseCapture{} + h.ClaimTasksByRuntime(rec, req) + status := rec.status + if status == 0 { + status = http.StatusOK + } + return status, json.RawMessage(rec.body.Bytes()), nil +} + +// newRPCClaimRequest rebuilds the HTTP claim request a WS connection stands in +// for. Every header the claim path reads must be forwarded from the identity +// captured at connect; Graph claims in particular bind an attempt to +// X-Daemon-Instance-ID and refuse a claim without it. +func newRPCClaimRequest(ctx context.Context, identity daemonws.ClientIdentity, body json.RawMessage) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/api/daemon/tasks/claim", bytes.NewReader(body)) + if err != nil { + return nil, err + } req.Header.Set("Content-Type", "application/json") if identity.UserID != "" { req.Header.Set("X-User-ID", identity.UserID) @@ -79,12 +98,8 @@ func (h *Handler) rpcClaimTasks(ctx context.Context, identity daemonws.ClientIde if identity.ClientVersion != "" { req.Header.Set("X-Client-Version", identity.ClientVersion) } - - rec := &rpcResponseCapture{} - h.ClaimTasksByRuntime(rec, req) - status := rec.status - if status == 0 { - status = http.StatusOK + if identity.InstanceID != "" { + req.Header.Set("X-Daemon-Instance-ID", identity.InstanceID) } - return status, json.RawMessage(rec.body.Bytes()), nil + return req, nil } diff --git a/server/internal/handler/daemon_rpc_test.go b/server/internal/handler/daemon_rpc_test.go index 3d92a37e65a5..d248b6017dd9 100644 --- a/server/internal/handler/daemon_rpc_test.go +++ b/server/internal/handler/daemon_rpc_test.go @@ -72,3 +72,38 @@ func TestDaemonRPCHandler_UnknownMethod(t *testing.T) { t.Fatalf("status=%d err=%v, want 404 + error", status, err) } } + +// TestNewRPCClaimRequestForwardsConnectionIdentity pins that the synthetic claim +// request carries every header the HTTP claim path reads. A Graph attempt binds +// to the daemon process identity; without X-Daemon-Instance-ID the claim is +// refused and the task is requeued in a loop. +func TestNewRPCClaimRequestForwardsConnectionIdentity(t *testing.T) { + identity := daemonws.ClientIdentity{ + UserID: "user-1", + Capabilities: "graph-execution-v1,graph-execution-v2", + ClientVersion: "1.2.3", + InstanceID: "6a1f6f0c-8f0b-4a37-8a0e-0a1d3f1d2b11", + } + req, err := newRPCClaimRequest(context.Background(), identity, json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + for header, want := range map[string]string{ + "X-User-ID": identity.UserID, + "X-Client-Capabilities": identity.Capabilities, + "X-Client-Version": identity.ClientVersion, + "X-Daemon-Instance-ID": identity.InstanceID, + } { + if got := req.Header.Get(header); got != want { + t.Errorf("%s = %q, want %q", header, got, want) + } + } + + bare, err := newRPCClaimRequest(context.Background(), daemonws.ClientIdentity{}, nil) + if err != nil { + t.Fatal(err) + } + if got := bare.Header.Get("X-Daemon-Instance-ID"); got != "" { + t.Errorf("empty identity must not invent an instance id, got %q", got) + } +} diff --git a/server/internal/handler/daemon_ws.go b/server/internal/handler/daemon_ws.go index 89d78f5f3195..ee6243bb3627 100644 --- a/server/internal/handler/daemon_ws.go +++ b/server/internal/handler/daemon_ws.go @@ -54,6 +54,7 @@ func (h *Handler) DaemonWebSocket(w http.ResponseWriter, r *http.Request) { RuntimeIDs: runtimeIDs, ClientVersion: r.Header.Get("X-Client-Version"), Capabilities: r.Header.Get("X-Client-Capabilities"), + InstanceID: r.Header.Get("X-Daemon-Instance-ID"), }) } diff --git a/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md b/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md index 3f8ca5f35aa7..3d4ea9c9115a 100644 --- a/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md +++ b/server/internal/service/builtin_skills/multica-working-on-issues/SKILL.md @@ -41,7 +41,8 @@ is a nonempty diagnostics object, for example the repository checkout. An unchanged result reuses a stable idempotency key. For `produce_commit`, commit locally and declare the exact `git_commit` artifact -described in the execution manifest and prompt; the daemon verifies and pushes +described in the execution manifest and prompt, with a freshly generated UUID as +its `id` (never the commit SHA); the daemon verifies and pushes the managed ref after execution stops. For `verify_commit`, keep HEAD at the frozen candidate, leave the checkout clean and return its `tested_commit_sha`. Do not replace structured submission with a comment, PR, exit code or a manual diff --git a/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md b/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md index 42446784f2e1..112151a622ed 100644 --- a/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md +++ b/server/internal/service/builtin_skills/multica-working-on-issues/references/working-on-issues-source-map.md @@ -18,7 +18,8 @@ - `server/internal/daemon/graph.go`: `buildGraphPrompt`, `finalizeGraphGit` and `finalizeGraphTask` define structured delivery and stopped receipts. - `server/internal/graph/result.go`: `ValidateResult` enforces execution-mode - artifacts, output schema, diagnostics and the tested candidate SHA. + artifacts (artifact ids must be UUIDs), output schema, diagnostics and the + tested candidate SHA. Evidence layer for `SKILL.md`. Every contract the skill states is traced to a current `file:line` here. Lines were re-derived against `feat/builtin-skills`