From aff6400dbb12c20fd5409317b62e73e57f11a59f Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 29 Oct 2026 22:01:05 +0800 Subject: [PATCH 1/2] fix: auto-retry Turns lost to a closed Runtime connection --- docs/configuration.md | 2 +- docs/zh/configuration.md | 4 ++-- services/core/internal/execution/turn_retry.go | 11 +++++++++++ services/core/internal/execution/turn_retry_test.go | 3 +++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bc2642a92..1e180a457 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -35,7 +35,7 @@ Model providers are not process settings; see [Default models](#default-models). | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | | `OAC_SANDBOX_MAX_ACTIVE` | `100` | Active sandbox limit for direct Providers with suspension, from 1 to 100000. Independent of execution concurrency and node capacity | | `OAC_SANDBOX_MAX_RETAINED` | `400` | Retained sandbox limit for direct Providers, from 1 to 100000, including active, suspended and unconfirmed cleanup. Must be at least the active limit | -| `OAC_TURN_AUTO_RETRIES` | `3` | Automatic retries of a Turn that failed on a recoverable model or tool error, from 0 to 5. `0` turns it off | +| `OAC_TURN_AUTO_RETRIES` | `3` | Automatic retries of a Turn that failed on a recoverable model or tool error, or because its Runtime connection closed, from 0 to 5. `0` turns it off | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 2e93192a6..730bbf433 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 7e6594fcab4b4fbf7d44d99e21745d106f200af902e2e7efacad5a7c19eea439 +source_hash: 900d0f3b1586911057c69291c8f17c951712297db26182a38d1b96a5563b5e12 --- Core 安装的每项设置、其归属位置以及修改方式。安装程序写入初始值；之后请使用本参考。每项设置都恰好只有一个归属位置，分属以下三类： @@ -39,7 +39,7 @@ Core 安装的每项设置、其归属位置以及修改方式。安装程序写 | `OAC_EXECUTION_CONCURRENCY` | `4` | Concurrent execution work, from 1 to 1024 | | `OAC_SANDBOX_MAX_ACTIVE` | `100` | 启用暂停能力的直接 Provider 的活跃沙箱上限，范围 1–100000；独立于执行并发度和节点容量 | | `OAC_SANDBOX_MAX_RETAINED` | `400` | 直接 Provider 的保留沙箱上限，范围 1–100000，包括活跃、已暂停以及尚未确认清理完成的分配；必须不小于活跃上限 | -| `OAC_TURN_AUTO_RETRIES` | `3` | Turn 因可恢复的模型或工具错误失败后的自动重试次数，0 到 5。设为 `0` 关闭 | +| `OAC_TURN_AUTO_RETRIES` | `3` | Turn 因可恢复的模型或工具错误、或因 Runtime 连接关闭而失败后的自动重试次数，0 到 5。设为 `0` 关闭 | | `OAC_DEFAULT_HARNESS` | `codex` | Harness used when a request does not name one | | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | diff --git a/services/core/internal/execution/turn_retry.go b/services/core/internal/execution/turn_retry.go index 2dd455e8d..2ffeccff3 100644 --- a/services/core/internal/execution/turn_retry.go +++ b/services/core/internal/execution/turn_retry.go @@ -73,6 +73,11 @@ func decideRetry(turn sessions.Turn, attempt, limit int) retryDecision { return skip("cancel_requested") case json.Unmarshal(turn.Outcome, &result) != nil: return skip("outcome_unreadable") + case runtimeDisconnected(result): + // The Runtime connection closed mid-Turn. The daemon stops the orphaned + // Turn before it reconnects, and the native history stays in the + // Environment, so a new Turn can continue from the last persisted step. + return retryDecision{Retry: true, Fingerprint: "runtime_disconnected"} case result.ErrorCode != "engine_failed": return skip("error_code_not_retryable") case !retryableEngineCodes[result.EngineErrorCode]: @@ -100,6 +105,12 @@ func decideRetry(turn sessions.Turn, attempt, limit int) retryDecision { return retryDecision{Retry: true, Fingerprint: name, Hint: retryHint(attempt+1, adjust)} } +// runtimeDisconnected reports a Turn lost only because its Runtime connection +// closed. Event window overflow and other stream failures are not retried. +func runtimeDisconnected(result Result) bool { + return result.ErrorCode == "event_stream_incomplete" && result.EventStreamError == "connection_closed" +} + // permanentHTTPStatus reports upstream statuses a new attempt cannot fix, which beta // can carry on a connection_failed classification. func permanentHTTPStatus(status *int) bool { diff --git a/services/core/internal/execution/turn_retry_test.go b/services/core/internal/execution/turn_retry_test.go index f6ec163e1..ebad6274e 100644 --- a/services/core/internal/execution/turn_retry_test.go +++ b/services/core/internal/execution/turn_retry_test.go @@ -43,6 +43,9 @@ func TestDecideRetryClasses(t *testing.T) { {"context length", engineFailure("context_length_exceeded", ""), false, ""}, {"cancelled", Result{ErrorCode: "cancelled"}, false, ""}, {"disconnected", Result{ErrorCode: "device_disconnected"}, false, ""}, + {"runtime connection closed", Result{ErrorCode: "event_stream_incomplete", EventStreamError: "connection_closed"}, true, "runtime_disconnected"}, + {"event window exceeded", Result{ErrorCode: "event_stream_incomplete", EventStreamError: "event_window_exceeded"}, false, ""}, + {"subscription failed", Result{ErrorCode: "event_stream_incomplete", EventStreamError: "subscription_failed"}, false, ""}, {"A", engineFailure("invalid_request", "400: Field messages is required"), true, "missing_messages"}, {"E", engineFailure("invalid_request", "Requests ending with a model turn are not supported"), true, "ends_with_model_turn"}, {"tool args", engineFailure("invalid_request", "tool_use ids were found without tool_result"), true, "tool_args"}, From f678e1765f06a6bee18f74ca54c63febb20caad6 Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 30 Oct 2026 09:17:24 +0800 Subject: [PATCH 2/2] fix: reject reasoning-only native completion and bound recovery --- .../agent/codex/completion_evidence.go | 43 ++++++++++ .../agent/codex/completion_evidence_test.go | 80 +++++++++++++++++++ apps/daemon/internal/agent/codex/session.go | 18 +++++ .../internal/agent/codex/session_messages.go | 4 + contracts/agents-api/core-errors.md | 1 + contracts/agents-api/zh/core-errors.md | 3 +- docs/runtime-protocol.md | 4 +- docs/zh/runtime-protocol.md | 6 +- internal/agentdaemon/proto/engine_failure.go | 2 +- .../agentdaemon/proto/engine_failure_test.go | 1 + services/core/deploy/codex/README.md | 4 + services/core/internal/execution/delivery.go | 2 +- .../core/internal/execution/turn_retry.go | 5 ++ .../internal/execution/turn_retry_test.go | 20 +++++ 14 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 apps/daemon/internal/agent/codex/completion_evidence.go create mode 100644 apps/daemon/internal/agent/codex/completion_evidence_test.go diff --git a/apps/daemon/internal/agent/codex/completion_evidence.go b/apps/daemon/internal/agent/codex/completion_evidence.go new file mode 100644 index 000000000..49d826600 --- /dev/null +++ b/apps/daemon/internal/agent/codex/completion_evidence.go @@ -0,0 +1,43 @@ +package codex + +import ( + "strings" + "sync" +) + +// completionEvidence observes root native items, even when reasoning is hidden. +// It does not decide business success or require artifacts from tool-only tasks. +type completionEvidence struct { + mu sync.Mutex + seen map[string]bool + last string + final bool +} + +func (e *completionEvidence) observe(item ThreadItem) { + e.mu.Lock() + defer e.mu.Unlock() + if e.seen[item.ID] { + return + } + if e.seen == nil { + e.seen = make(map[string]bool) + } + e.seen[item.ID] = true + // Only finite types reach logs; an unknown item prevents a positive diagnosis. + switch item.Type { + case "reasoning", "agentMessage", "commandExecution", "fileChange", "mcpToolCall", "dynamicToolCall", "webSearch": + e.last = item.Type + default: + e.last = "other" + } + if item.Type == "agentMessage" && (item.Phase == "final_answer" || item.Phase == "") && strings.TrimSpace(item.Text) != "" { + e.final = true + } +} + +func (e *completionEvidence) snapshot() (string, bool) { + e.mu.Lock() + defer e.mu.Unlock() + return e.last, e.final +} diff --git a/apps/daemon/internal/agent/codex/completion_evidence_test.go b/apps/daemon/internal/agent/codex/completion_evidence_test.go new file mode 100644 index 000000000..3ff12bd38 --- /dev/null +++ b/apps/daemon/internal/agent/codex/completion_evidence_test.go @@ -0,0 +1,80 @@ +package codex + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" +) + +func TestNativeCompletionEvidence(t *testing.T) { + for _, tc := range []struct { + name string + items []ThreadItem + status string + fail bool + }{ + {"reasoning only", []ThreadItem{{ID: "r", Type: "reasoning"}}, "completed", true}, + {"early commentary", []ThreadItem{{ID: "a", Type: "agentMessage", Phase: "commentary", Text: "SECRET"}, {ID: "r", Type: "reasoning"}}, "completed", true}, + {"tool then reasoning", []ThreadItem{{ID: "t", Type: "commandExecution", Command: "true", Status: "completed", ExitCode: new(int)}, {ID: "r", Type: "reasoning"}}, "completed", true}, + {"final answer", []ThreadItem{{ID: "r", Type: "reasoning"}, {ID: "a", Type: "agentMessage", Phase: "final_answer", Text: "SECRET"}}, "completed", false}, + {"unphased answer", []ThreadItem{{ID: "a", Type: "agentMessage", Text: "SECRET"}, {ID: "r", Type: "reasoning"}}, "completed", false}, + {"post final reasoning", []ThreadItem{{ID: "a", Type: "agentMessage", Phase: "final_answer", Text: "SECRET"}, {ID: "r", Type: "reasoning"}}, "completed", false}, + {"tool only", []ThreadItem{{ID: "t", Type: "commandExecution", Command: "true", Status: "completed", ExitCode: new(int)}}, "completed", false}, + {"empty", nil, "completed", false}, + {"unknown", []ThreadItem{{ID: "r", Type: "reasoning"}, {ID: "x", Type: "SECRET"}}, "completed", false}, + {"duplicate reasoning", []ThreadItem{{ID: "r", Type: "reasoning"}, {ID: "a", Type: "agentMessage", Text: "SECRET"}, {ID: "r", Type: "reasoning"}}, "completed", false}, + {"interrupted", []ThreadItem{{ID: "r", Type: "reasoning"}}, "interrupted", false}, + } { + for _, visible := range []bool{true, false} { + t.Run(tc.name+map[bool]string{true: " visible", false: " hidden"}[visible], func(t *testing.T) { + var logs bytes.Buffer + cfg := defaultSessionConfig(testStateRoot(t)) + cfg.logger = slog.New(slog.NewJSONHandler(&logs, nil)) + out := make(chan proto.Envelope, 64) + s := &Session{runID: "run", out: out, cancelCtx: context.Background(), cfg: cfg} + s.reasoningSummary = visible + s.setThreadID("thread") + s.beginRootTurn("thread", "turn") + for _, item := range tc.items { + raw, _ := json.Marshal(ItemCompletedNotification{ThreadID: "thread", TurnID: "turn", Item: item}) + s.onItemCompleted(raw) + s.beginRootTurn("thread", "turn") // duplicate start must retain evidence + } + // Child and stale turn evidence cannot change the root outcome. + s.onItemCompleted(json.RawMessage(`{"threadId":"child","turnId":"turn","item":{"id":"child","type":"reasoning"}}`)) + s.onItemCompleted(json.RawMessage(`{"threadId":"thread","turnId":"old","item":{"id":"old","type":"agentMessage","phase":"final_answer","text":"SECRET"}}`)) + raw, _ := json.Marshal(TurnCompletedNotification{ThreadID: "thread", Turn: Turn{ID: "turn", Status: tc.status}}) + s.onTurnCompleted(raw) + s.onTurnCompleted(raw) + failures, done := 0, 0 + for env := range out { + if env.Type == proto.TypeError { + var p proto.ErrorPayload + _ = env.DecodePayload(&p) + if p.Code != "incomplete_response" { + t.Fatalf("code %q", p.Code) + } + failures++ + } + if env.Type == proto.TypeDone { + done++ + } + } + if (failures == 1) != tc.fail || done != 1 { + t.Fatalf("failures=%d done=%d", failures, done) + } + if strings.Contains(logs.String(), "SECRET") { + t.Fatal("content leaked") + } + if !strings.Contains(logs.String(), `"decision_source":"codex_adapter"`) { + t.Fatal("missing decision source") + } + }) + } + } +} diff --git a/apps/daemon/internal/agent/codex/session.go b/apps/daemon/internal/agent/codex/session.go index 207514baa..43788f4a9 100644 --- a/apps/daemon/internal/agent/codex/session.go +++ b/apps/daemon/internal/agent/codex/session.go @@ -102,6 +102,7 @@ type Session struct { // reasoningSummary reports that the Agent requested reasoning summaries. reasoningSummary bool + completion completionEvidence finalTextMu sync.Mutex finalText string lastErrText string @@ -226,6 +227,14 @@ func (s *Session) onTurnCompleted(raw json.RawMessage) { } status := strings.ToLower(p.Turn.Status) + lastItem, hasFinal := s.completion.snapshot() + reason := "native_terminal" + incomplete := status == "completed" && lastItem == "reasoning" && !hasFinal + if incomplete { + reason = "reasoning_only_tail" + } else if status == "failed" { + reason = "native_failed" + } finalText := s.takeFinalText() errText := s.takeLastErrText() // Always log the turn outcome — operators need this when a prompt @@ -237,9 +246,18 @@ func (s *Session) onTurnCompleted(raw json.RawMessage) { "run_id", s.runID, "turn_id", p.Turn.ID, "status", p.Turn.Status, + "decision_source", "codex_adapter", + "decision_reason", reason, + "last_completed_item_type", lastItem, + "has_final_answer", hasFinal, "final_text_len", len(finalText), "buffered_err_text_len", len(errText), "has_error", p.Turn.Error != nil) + if incomplete { + s.emitTerminalFailure("Native execution ended with reasoning but no final response or subsequent action.", true, proto.ErrorPayload{Code: "incomplete_response"}) + s.finishAfterTerminal() + return + } if status == "failed" { // Body precedence on failure: // 1. agent's final text (rare on hard failures but exists for diff --git a/apps/daemon/internal/agent/codex/session_messages.go b/apps/daemon/internal/agent/codex/session_messages.go index 4d7642f36..6d1fea7e7 100644 --- a/apps/daemon/internal/agent/codex/session_messages.go +++ b/apps/daemon/internal/agent/codex/session_messages.go @@ -59,6 +59,7 @@ func (s *Session) onItemCompleted(raw json.RawMessage) { return } if p.Item.Type == "reasoning" { + s.completion.observe(p.Item) s.completeReasoning(p.Item) return } @@ -72,6 +73,9 @@ func (s *Session) onItemCompleted(raw json.RawMessage) { if messageText == "" { messageText = text } + completedItem := p.Item + completedItem.Text = messageText + s.completion.observe(completedItem) first := false if p.Item.Type == "agentMessage" && messageText != "" { first = s.boundary.text(received, p.TurnID) diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md index e6ebbe4e2..378d7ec96 100644 --- a/contracts/agents-api/core-errors.md +++ b/contracts/agents-api/core-errors.md @@ -122,6 +122,7 @@ The [Session diagnostics](./session-diagnostics.md) routes return these categori | `request_timeout` | Native provider timeout classification | | `context_length_exceeded` | Native context limit classification | | `cyber_policy` | Native cyber policy rejection | +| `incomplete_response` | The executor terminated without the expected response; params are empty and the public Turn error maps to `internal_error` | | `connection_failed` | Native connection failure; params contain `http_status`, an integer in 100–599 or null | | `model_provider_required` | Missing frozen model provider | | `runtime_unavailable` | `execution_device_unavailable`, `execution_unavailable` | diff --git a/contracts/agents-api/zh/core-errors.md b/contracts/agents-api/zh/core-errors.md index 3d6831fde..8eff293ad 100644 --- a/contracts/agents-api/zh/core-errors.md +++ b/contracts/agents-api/zh/core-errors.md @@ -1,7 +1,7 @@ --- title: "Core 管理错误" source: contracts/agents-api/core-errors.md -source_hash: eeaba03a33dd1f82943139292035873d8139a5dd862d82e1f9cf1267f1ad10c6 +source_hash: 320fe54d2db2fb022f2d6069daf74c5b189c87db3575a36c58242bf98d7fea61 --- `/core/v1` 上的错误使用此封装结构。`message` 是安全的英文文本；`code` 和 `param` 可以为 null。客户端依据稳定的 `code` 和可选的 `param` 进行处理，对未知代码显示 `message`，绝不解析消息，也绝不自动重试被拒绝的写操作。 @@ -124,6 +124,7 @@ Web 的控制台服务器在 `/core` 路径上发生自身故障时使用此封 | `request_timeout` | 原生提供商超时分类 | | `context_length_exceeded` | 原生上下文限制分类 | | `cyber_policy` | 原生网络安全策略拒绝 | +| `incomplete_response` | 执行器已终止但缺少预期响应；params 为空，公共 Turn 错误映射为 `internal_error` | | `connection_failed` | 原生连接故障；params 包含 `http_status`，其值为 100–599 范围内的整数或 null | | `model_provider_required` | 缺少已冻结的模型提供商 | | `runtime_unavailable` | `execution_device_unavailable`、`execution_unavailable` | diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index 19e9365dd..f53f65051 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -306,10 +306,12 @@ Retry only an operation whose own contract makes retry safe. A retired preparati An adapter may add `code` and `http_status` to a Run's `error` frame. They are optional neutral metadata, not a terminal event; the frame's text, Usage, `done`, native Session identity and cancellation receipts keep their order and meaning. -The accepted codes are `authentication_error`, `rate_limit_exceeded`, `usage_limit_exceeded`, `server_overloaded`, `server_error`, `invalid_request`, `resource_not_found`, `request_timeout`, `context_length_exceeded`, `cyber_policy`, `provider_billing_unavailable` and `connection_failed` ([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)). `connection_failed` keeps an integer `http_status` from 100 to 599; status 402 normalizes to `provider_billing_unavailable`, which keeps only status 402. This category describes the model endpoint billing rejection, not the application user balance, and does not authorize automatic retries. Every other status is discarded. A missing, malformed or unknown value leaves the error unclassified without discarding Usage or `done`. +The accepted codes are `authentication_error`, `rate_limit_exceeded`, `usage_limit_exceeded`, `server_overloaded`, `server_error`, `invalid_request`, `resource_not_found`, `request_timeout`, `context_length_exceeded`, `cyber_policy`, `provider_billing_unavailable`, `incomplete_response` and `connection_failed` ([`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)). `connection_failed` keeps an integer `http_status` from 100 to 599; status 402 normalizes to `provider_billing_unavailable`, which keeps only status 402. This category describes the model endpoint billing rejection, not the application user balance, and does not authorize automatic retries. Every other status is discarded. A missing, malformed or unknown value leaves the error unclassified without discarding Usage or `done`. Core stores accepted values in the Turn outcome as `engine_error_code` and `engine_http_status`. Within a Turn, a later accepted classification replaces the earlier classification and its status; an unclassified error frame does not erase them. Classification does not carry over to another Turn. The optional `upstream_request_id` is accepted only as a UUID and stored privately as `engine_request_id`; it never enters Project diagnostics, public Session/Turn responses or SSE. The classification is subordinate to the terminal status and Core's `error_code`: it cannot turn a completed or cancelled Turn into a failure, hide an incomplete event stream, or override a persistence or cancellation-receipt failure. Normal delivery and terminal journal draining use the same extraction. [Session diagnostics](../contracts/agents-api/session-diagnostics.md) expose the category only for a failed Turn whose Core error is `engine_failed`. The [Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) and [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter guides give each Harness's mapping; adapters classify structured Harness errors and model provider JSON error bodies using their accepted `error.code` or `error.type`, never arbitrary error prose. Codex alone recognizes its exact terminal HTTP 402 envelope when structured information is absent or `other`; see its adapter guide for the bounded rule. +`incomplete_response` means the adapter observed a native terminal sequence that ended without the expected response or subsequent action. It is not a business-delivery verdict. Core preserves this typed failure, allows at most one automatic continuation within the configured Turn retry budget, and retains cancellation and newest-Turn checks. The continuation asks the Harness to inspect existing results and avoid repeating completed side effects; external operations still require application-owned idempotency. Codex detects a completed root Turn whose last unique completed item is reasoning and which has no nonempty `final_answer` or unphased assistant message; earlier commentary does not satisfy that check. Hidden reasoning is still observed. Unknown tail types, tool-only completions and ordinary empty completions do not activate this rule. + ## Workspace operations A workspace read that needs no running Turn uses the read-only preparation profile: `execution_prepare` with `workspace_read_only`, which requires the `workspace_read_preparation` capability. It accepts only the bound Environment and resource identity; execution options, model and MCP credentials, native Session continuation and model or tool input are excluded, and the owner rejects `execution_start`. The Runtime serves it from its bound local workspace without starting a Harness process. The profile publishes `released` after the Runtime drops the preparation's ownership; a stale status snapshot never publishes success. A release request, HTTP disconnect or remote socket closure alone does not confirm the release. diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index 088dbbaaa..a33c3e27d 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 89038da4d9b924fe2bd0e28dd2420515e8d9cf38988dd53e1fd9370d66655a75 +source_hash: f1c9e5c733e490c58a93080a0cca96904d43b38b897acbbabac04e701f6d6126 --- Core–Runtime 协议连接 Core 与 Runtime daemon，定义 daemon 连接上消息的含义和顺序。托管和自托管 Runtime 使用同一协议。 @@ -308,10 +308,12 @@ transport 与 execution 结果分开。Core 唯一 Run 订阅入口是 `Subscrib adapter 可以给 Run 的 `error` frame 添加 `code` 和 `http_status`。它们是可选的中性元数据，不是终结事件；frame 文本、Usage、`done`、原生 Session 身份和取消回执保持原顺序与含义。 -接受的 code 为 `authentication_error`、`rate_limit_exceeded`、`usage_limit_exceeded`、`server_overloaded`、`server_error`、`invalid_request`、`resource_not_found`、`request_timeout`、`context_length_exceeded`、`cyber_policy`、`provider_billing_unavailable` 和 `connection_failed`（[`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)）。`connection_failed` 保留 100 到 599 的整数 `http_status`；402 归一为 `provider_billing_unavailable`，此分类仅保留状态 402。它表示模型端点的计费拒绝，不代表应用用户余额不足，也不授权自动重试。其他 status 都丢弃。值缺失、格式错误或未知时保持未分类错误，不丢弃 Usage 或 `done`。 +接受的 code 为 `authentication_error`、`rate_limit_exceeded`、`usage_limit_exceeded`、`server_overloaded`、`server_error`、`invalid_request`、`resource_not_found`、`request_timeout`、`context_length_exceeded`、`cyber_policy`、`provider_billing_unavailable`、`incomplete_response` 和 `connection_failed`（[`engine_failure.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/agentdaemon/proto/engine_failure.go)）。`connection_failed` 保留 100 到 599 的整数 `http_status`；402 归一为 `provider_billing_unavailable`，此分类仅保留状态 402。它表示模型端点的计费拒绝，不代表应用用户余额不足，也不授权自动重试。其他 status 都丢弃。值缺失、格式错误或未知时保持未分类错误，不丢弃 Usage 或 `done`。 Core 在 Turn outcome 中将接受的值保存为 `engine_error_code` 和 `engine_http_status`。同一 Turn 内，后续有效分类会替换此前分类及其状态码；未分类的 error frame 不会清空它们。分类不会沿用到其他 Turn。可选 `upstream_request_id` 仅接受 UUID，并私有保存为 `engine_request_id`；不得进入 Project 诊断、公开 Session/Turn 响应或 SSE。分类从属于终结状态和 Core 的 `error_code`：不能将已完成或已取消 Turn 变成失败、隐藏不完整事件流，或覆盖持久化或取消回执失败。正常 delivery 和 terminal journal draining 使用同一提取逻辑。[Session 诊断](../../contracts/agents-api/zh/session-diagnostics.md)仅为 Core error 为 `engine_failed` 的失败 Turn 暴露类别。[Codex](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/codex/README.md) 和 [Claude Code](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/deploy/claude/README.md) adapter 指南给出各 Harness 的映射；adapter 根据结构化 Harness 错误及模型提供商 JSON 错误体中受支持的 `error.code` 或 `error.type` 分类，不依据任意错误文字。仅 Codex 在结构化信息缺失或为 `other` 时识别准确的终态 HTTP 402 封装；严格边界见该适配器指南。 +`incomplete_response` 表示适配器观察到原生终止序列缺少预期响应或后续动作，不代表业务交付验收。Core 保留此分类，并在配置的 Turn 重试预算内最多自动续跑一次；取消与最新 Turn 检查仍然生效。续跑要求 Harness 先检查已有结果，避免重复已完成的副作用；外部操作仍需应用自行实现幂等。Codex 检测状态为 completed、最后一个去重后的已完成根条目为 reasoning、且没有非空 `final_answer` 或未标注 phase 的助手消息 的 Turn；早期 commentary 不能满足该检查。隐藏的 reasoning 仍被观察。未知末尾类型、仅工具完成和普通空完成不会触发此规则。 + ## 工作区操作 {#workspace-operations} 无需运行 Turn 的工作区读取使用只读 preparation profile：带 `workspace_read_only` 的 `execution_prepare`，要求 `workspace_read_preparation` 能力。仅接受绑定的 Environment 和 resource 身份；不包含 execution option、model 与 MCP 凭据、原生 Session continuation、model 或 tool 输入，owner 拒绝 `execution_start`。Runtime 从绑定的本地工作区提供读取，不启动 Harness 进程。profile 在 Runtime 放弃该 preparation 的所有权后发布 `released`；旧 status snapshot 不发布成功。release 请求、HTTP 断连或远端 socket 关闭本身都不确认释放。 diff --git a/internal/agentdaemon/proto/engine_failure.go b/internal/agentdaemon/proto/engine_failure.go index 0239089df..331d730da 100644 --- a/internal/agentdaemon/proto/engine_failure.go +++ b/internal/agentdaemon/proto/engine_failure.go @@ -19,7 +19,7 @@ func NormalizeProviderRequestID(id string) string { // metadata from newer or malformed peers never changes terminal/usage decoding. func NormalizeEngineFailure(code string, status *int) (string, *int) { switch code { - case "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy": + case "incomplete_response", "authentication_error", "rate_limit_exceeded", "usage_limit_exceeded", "server_overloaded", "server_error", "invalid_request", "resource_not_found", "request_timeout", "context_length_exceeded", "cyber_policy": return code, nil case "connection_failed": if status != nil && *status >= 100 && *status <= 599 { diff --git a/internal/agentdaemon/proto/engine_failure_test.go b/internal/agentdaemon/proto/engine_failure_test.go index df4436866..60f85c45a 100644 --- a/internal/agentdaemon/proto/engine_failure_test.go +++ b/internal/agentdaemon/proto/engine_failure_test.go @@ -12,6 +12,7 @@ func TestErrorClassificationCompatibility(t *testing.T) { }{ {`{"error":"old"}`, "", 0}, {`{"error":"old","code":"authentication_error","http_status":401}`, "authentication_error", 0}, + {`{"error":"old","code":"incomplete_response","http_status":503}`, "incomplete_response", 0}, {`{"error":"old","code":"connection_failed","http_status":503}`, "connection_failed", 503}, {`{"error":"old","code":"connection_failed","http_status":"secret"}`, "connection_failed", 0}, {`{"error":"old","code":"connection_failed","http_status":99}`, "connection_failed", 0}, diff --git a/services/core/deploy/codex/README.md b/services/core/deploy/codex/README.md index 14586b75d..1691a1a84 100644 --- a/services/core/deploy/codex/README.md +++ b/services/core/deploy/codex/README.md @@ -77,3 +77,7 @@ Codex `0.153.4` reports a provider HTTP 400 as `other` and puts the provider's e The build fails unless `codex --version` reports the pinned version. The image holds no credentials, workspace data or product software. The distribution copies the Codex executable and resources into the combined Runtime image; see the [maintainer guide](../../../../docs/maintainers.md#runtime-images-and-helpers). The [Docker adapter](../../../../docs/sandbox-provider.md#docker-adapter) owns the shared container lifecycle and isolation settings. + +### Completion evidence + +The adapter observes root completed item types independently of reasoning visibility. A native `completed` Turn ending with a unique reasoning item and no nonempty `final_answer` or unphased assistant message emits `incomplete_response` before `done`, rather than accepting earlier commentary as completion. This is a bounded-failure override for unattended execution; it does not validate business deliverables. Duplicate item completions do not change the tail, and child or stale Turn items do not contribute. Tool-only and empty completions retain native semantics. The turn/completed log records `decision_source`, `decision_reason`, `last_completed_item_type` and `has_final_answer` without item bodies. Core logs the received Runtime terminal source and any preceding failure, then separately logs the confirmed persisted status. See [the Runtime protocol](../../../../docs/runtime-protocol.md#native-failure-classification) for classification and bounded continuation. diff --git a/services/core/internal/execution/delivery.go b/services/core/internal/execution/delivery.go index 2de10d1e5..84f93f7fa 100644 --- a/services/core/internal/execution/delivery.go +++ b/services/core/internal/execution/delivery.go @@ -222,7 +222,7 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe nativeObserved = true if env.Type == proto.TypeDone && doneReceived.IsZero() { doneReceived = time.Now() - obslog.Ctx(ctx).Info("executor done received", "session_id", sessionID, "turn_id", runID) + obslog.Ctx(ctx).Info("executor done received", "session_id", sessionID, "turn_id", runID, "decision_source", "runtime_done", "prior_error_code", result.ErrorCode, "engine_error_code", result.EngineErrorCode, "business_delivery_verified", false) } if !firstTextObserved && hasText(env) { firstTextObserved = true diff --git a/services/core/internal/execution/turn_retry.go b/services/core/internal/execution/turn_retry.go index 2ffeccff3..0b56bae30 100644 --- a/services/core/internal/execution/turn_retry.go +++ b/services/core/internal/execution/turn_retry.go @@ -80,6 +80,11 @@ func decideRetry(turn sessions.Turn, attempt, limit int) retryDecision { return retryDecision{Retry: true, Fingerprint: "runtime_disconnected"} case result.ErrorCode != "engine_failed": return skip("error_code_not_retryable") + case result.EngineErrorCode == "incomplete_response": + if attempt >= 1 { + return skip("incomplete_response_attempts_exhausted") + } + return retryDecision{Retry: true, Fingerprint: "incomplete_response", Hint: "Inspect the existing results first. Continue only missing work and provide a final response. Do not repeat completed side effects; verify uncertain submissions before retrying them."} case !retryableEngineCodes[result.EngineErrorCode]: return skip("engine_code_not_retryable") case permanentHTTPStatus(result.EngineHTTPStatus): diff --git a/services/core/internal/execution/turn_retry_test.go b/services/core/internal/execution/turn_retry_test.go index ebad6274e..8d3fe1304 100644 --- a/services/core/internal/execution/turn_retry_test.go +++ b/services/core/internal/execution/turn_retry_test.go @@ -207,3 +207,23 @@ func TestAutoRetrySkips(t *testing.T) { t.Fatal("disabled switch still retried") } } + +func TestIncompleteResponseRetryBound(t *testing.T) { + turn := failedTurn(t, engineFailure("incomplete_response", "private native text")) + for _, tc := range []struct { + attempt, limit int + retry bool + }{{0, 0, false}, {0, 3, true}, {1, 3, false}, {2, 5, false}} { + d := decideRetry(turn, tc.attempt, tc.limit) + if d.Retry != tc.retry { + t.Fatalf("%+v: %+v", tc, d) + } + if d.Retry && (!strings.Contains(d.Hint, "Do not repeat completed side effects") || strings.Contains(retryMessage(1, 3, d), "private native text")) { + t.Fatal("unsafe recovery hint") + } + } + turn.CancelRequestedAt = time.Now() + if decideRetry(turn, 0, 3).Retry { + t.Fatal("cancelled turn retried") + } +}