From cef4e23a72515ce388ea096470a2128ca78e6c7e Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 20 Nov 2026 08:12:57 +0900 Subject: [PATCH 1/3] Archive model calls in self-contained, optionally age-encrypted trace segments --- cmd/server/main.go | 7 +- config.example.yaml | 15 + config.hub.example.yaml | 15 + docs/trace-archive.md | 95 ++++ go.mod | 1 + go.sum | 2 + internal/api/handlers/management/config_v8.go | 12 + internal/api/server.go | 6 + internal/api/server_management_v8.go | 4 + internal/api/server_reload.go | 1 + internal/api/server_trace_archive.go | 101 +++++ internal/api/server_trace_archive_test.go | 236 ++++++++++ internal/config/config.go | 1 + internal/config/config_load.go | 3 + internal/config/config_v8.go | 2 +- internal/config/parse.go | 3 + internal/config/trace_archive.go | 40 ++ internal/doctor/config.go | 7 + .../executor/codex_websockets_duplex.go | 44 +- .../executor/codex_websockets_stream.go | 3 + .../runtime/executor/helps/logging_helpers.go | 8 + .../runtime/executor/helps/usage_helpers.go | 6 + internal/tracearchive/archive_test.go | 281 ++++++++++++ internal/tracearchive/capture.go | 383 ++++++++++++++++ internal/tracearchive/cli.go | 215 +++++++++ internal/tracearchive/json.go | 207 +++++++++ internal/tracearchive/recorder.go | 413 +++++++++++++++++ internal/tracearchive/response.go | 310 +++++++++++++ internal/tracearchive/segment.go | 423 ++++++++++++++++++ internal/tracearchive/upload.go | 230 ++++++++++ sdk/api/handlers/handlers.go | 3 + .../openai/openai_responses_websocket.go | 11 + sdk/cliproxy/usage/manager.go | 11 + .../components/sections/SectionLogging.tsx | 75 +++- web/src/features/config/constants.ts | 2 + web/src/features/config/searchIndex.ts | 14 + .../features/config/visualConfigAdditions.ts | 2 + web/src/i18n/locales/en.json | 7 + web/src/types/visualConfig.ts | 4 + 39 files changed, 3193 insertions(+), 10 deletions(-) create mode 100644 docs/trace-archive.md create mode 100644 internal/api/server_trace_archive.go create mode 100644 internal/api/server_trace_archive_test.go create mode 100644 internal/config/trace_archive.go create mode 100644 internal/tracearchive/archive_test.go create mode 100644 internal/tracearchive/capture.go create mode 100644 internal/tracearchive/cli.go create mode 100644 internal/tracearchive/json.go create mode 100644 internal/tracearchive/recorder.go create mode 100644 internal/tracearchive/response.go create mode 100644 internal/tracearchive/segment.go create mode 100644 internal/tracearchive/upload.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 7d41142842e..5606a795a01 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,6 +17,8 @@ import ( "strings" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/joho/godotenv" configaccess "github.com/rune-oai/t3code-proxy/internal/access/config_access" "github.com/rune-oai/t3code-proxy/internal/api" @@ -89,6 +91,9 @@ func run() (exitStatus int) { // "doctor" checks whether this machine is ready to act as a hub. It needs // no running server and never reads credentials. + if len(os.Args) > 1 && os.Args[1] == "traces" { + return tracearchive.Main(os.Args[2:], os.Stdout, os.Stderr) + } if len(os.Args) > 1 && os.Args[1] == "doctor" { return doctor.Main(context.Background(), os.Args[2:], os.Stdout, os.Stderr) } @@ -190,7 +195,7 @@ func run() (exitStatus int) { flag.CommandLine.Usage = func() { out := flag.CommandLine.Output() _, _ = fmt.Fprintf(out, "Usage of %s\n", os.Args[0]) - _, _ = fmt.Fprint(out, "\nCommands:\n doctor Check whether this machine is ready to run as a hub (doctor -h for flags)\n discover Find gateways on the local network (discover -h for flags)\n\nFlags:\n") + _, _ = fmt.Fprint(out, "\nCommands:\n doctor Check whether this machine is ready to run as a hub (doctor -h for flags)\n discover Find gateways on the local network (discover -h for flags)\n traces Export, verify or summarize archived calls\n\nFlags:\n") flag.CommandLine.VisitAll(func(f *flag.Flag) { if f.Name == "password" { return diff --git a/config.example.yaml b/config.example.yaml index 0f1643c601a..0982cedbde2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -9,6 +9,21 @@ # The compatibility notes at the end show old spellings for existing deployments. config-version: 8 +# Durable model-call history. Live files contain private conversation data. +# See docs/trace-archive.md for encryption, recovery, and export. +# trace-archive: +# enabled: false +# dir: "" # default: /traces +# recipients: [] # age1... X25519 public keys +# queue-size: 32 +# max-local-size-mb: 0 # zero disables local eviction +# exclude-endpoints: [] +# exclude-models: [] +# upload: +# enabled: false +# binary: rclone +# remote: gdrive:t3code-proxy-traces + # Optional model catalog sources: http(s) URLs or absolute local file paths. # Each explicit source overrides --local-model; otherwise --local-model uses # embedded catalogs, and normal mode uses the official remote sources. diff --git a/config.hub.example.yaml b/config.hub.example.yaml index 982150690fa..7b1769e1ae3 100644 --- a/config.hub.example.yaml +++ b/config.hub.example.yaml @@ -5,6 +5,21 @@ # documented in config.example.yaml. config-version: 8 +# Durable model-call history. Live files contain private conversation data. +# See docs/trace-archive.md for encryption, recovery, and export. +# trace-archive: +# enabled: false +# dir: "" # default: /traces +# recipients: [] # age1... X25519 public keys +# queue-size: 32 +# max-local-size-mb: 0 # zero disables local eviction +# exclude-endpoints: [] +# exclude-models: [] +# upload: +# enabled: false +# binary: rclone +# remote: gdrive:t3code-proxy-traces + server: # Listen on loopback only. Tailscale Serve publishes the port to your tailnet: # tailscale serve --bg --https=8317 http://127.0.0.1:8317 diff --git a/docs/trace-archive.md b/docs/trace-archive.md new file mode 100644 index 00000000000..c3ed9d8b6a5 --- /dev/null +++ b/docs/trace-archive.md @@ -0,0 +1,95 @@ +# Trace archive + +Enable `trace-archive.enabled` to keep model calls for evaluation, replay or training. The default is off. The archive works independently of request logging. The web panel provides the switch, directory and recorder status under Logging. Set encryption and upload options in YAML. Settings reload with the hub configuration. + +## Privacy and keys + +A trace includes instructions, conversation history, tool arguments, tool results, generated content and whatever secrets a user pasted into a session. The live file is plaintext. Use a private directory on an encrypted volume, restrict its OS permissions to the hub account, and protect exports just as carefully. New directories use mode 0700 and new files use 0600 on POSIX. Set Windows ACLs on the parent directory. + +Generate an age identity on the machine where you will read the archives. Keep the private identity there. Back it up somewhere other than the upload destination. Losing every matching identity makes a sealed encrypted segment unrecoverable. + +```sh +age-keygen -o trace-identity.txt +age-keygen -y trace-identity.txt +``` + +Copy only the printed `age1...` public recipient to the hub. Any configured recipient can decrypt. A segment uses the recipients configured when it seals. A hub with public keys alone can read its live file and cannot decrypt sealed history. Omitting recipients produces an unencrypted zstd file and a warning at seal time. + +## Configuration and Google Drive + +```yaml +trace-archive: + enabled: true + dir: /srv/private/traces + recipients: [age1REPLACE_WITH_PUBLIC_RECIPIENT] + queue-size: 32 + max-local-size-mb: 8192 + exclude-models: [] + exclude-endpoints: [] + upload: + enabled: true + binary: rclone + remote: gdrive:t3code-proxy-traces +``` + +1. Install rclone for the OS user that runs the hub. Run `rclone config`, create a remote named `gdrive`, select Google Drive and authorize the account. Keep rclone's configuration private. +2. Run `rclone mkdir gdrive:t3code-proxy-traces` and `rclone lsf gdrive:t3code-proxy-traces` as the hub user. +3. Apply the config and make a test call. Send an authenticated POST to `/v8/management/trace-archive/seal-now`. Check `GET /v8/management/trace-archive/status` for the segment, verified destination, dropped count and errors. +4. Test recovery on the identity machine. Copy a segment and its matching manifest into one directory, then run `traces verify` and `traces export`. + +The default directory is `traces` below the resolved auth directory. Filters are exact client model names and URL paths. Queue size bounds concurrent captures plus queued calls. A call exceeding the available slots, 128 MiB per call, 256 MiB of aggregate capture buffers, or 256 levels of JSON nesting is dropped whole. Disk, compression, reconstruction, encryption and networking run outside client handlers. Counters for dropped calls reset when the hub restarts. The local limit is in MiB and zero means unlimited. Retention removes the oldest sealed pairs first. With uploads enabled, only a pair verified at the configured destination qualifies. The live segment always stays on disk. + +Both files are sent with `rclone copyto`. The hub streams each back with `rclone cat` to check the exact byte length and SHA-256 even on remotes that expose only MD5. This uses bandwidth in both directions. A verified pair has the destination in local and remote manifest field `uploaded_to`. Errors retry in the background with exponential backoff up to one hour. Uploads are cancellable at shutdown. + +## Reading a segment without the hub + +```sh +age -d -i trace-identity.txt traces-2026-11-20-123.jsonl.zst.age | zstd -dc > restored.jsonl +jq -c 'select(.type == "call") | .call.response' restored.jsonl +``` + +Schema version 1 uses UTF-8 JSON objects separated by LF, one object per line, including the final LF. Dates are UTC. File names are `traces-YYYY-MM-DD[-n].live` while open and `traces-YYYY-MM-DD[-n].jsonl.zst[.age]` once sealed. The writer uses a nanosecond suffix to distinguish same-day segments. The date is the recording day. A new day or a manual seal starts an independent blob dictionary. A current-day live file can survive a clean shutdown. Recovery truncates only the unterminated final line, rejects a corrupt complete line and seals stale live files. + +Entries appear in this order: + +* A header has `type: "header"`, integer `schema_version: 1` and `date`. +* A blob has `type: "blob"`, `sha256` (lowercase hex) and `value` (any JSON value). Hash the UTF-8 canonical JSON value. Objects sort keys by their UTF-8 byte strings recursively, arrays retain order, and separators have no whitespace. Strings use UTF-8 in double quotes. Escape quotes, backslashes and the five short control escapes. Encode other ASCII control characters, `<`, `>` and `&` as lowercase `\u00xx`, and U+2028/U+2029 as `\u2028`/`\u2029`. Numeric literals retain their original spelling, including large integers. Each hash appears once and before its first reference. +* A call has `type: "call"` and a `call` object with its own integer `schema_version`. Its packed `client_request` and `upstream_request` each have a `body` and optional `parts`. Copy body, then insert each parts entry at the same top-level key. A string denotes one blob hash. An array denotes an ordered array of blob hashes. Empty arrays remain empty. Fields eligible for extraction are system, instructions, tools, messages, input, contents, systemInstruction and toolConfig. The `base64` alternative holds a non-JSON request as standard padded Base64. Missing payload fields mean that side was unobserved. +* The call's `response` is the reconstructed upstream object. Claude thinking, signatures, redacted blocks, tools, citations and stop information stay in provider format. Responses items retain opaque encrypted content and summaries. Chat chunks become choices with complete messages and tool calls. For HTTP calls, a differing final client result is stored as `client_response`, or `client_raw_base64` if it cannot be reconstructed. The client-facing body supplies a fallback when upstream capture is unavailable. When reconstruction fails, `raw_frames_base64` contains ordered standard Base64 snapshots of executor chunks (scanner lines, HTTP bodies or WebSocket messages). Base64 decoding recovers each frame independently. Network headers are omitted. Recognized credential fields inside JSON header and metadata containers are masked. +* Metadata includes hub version, client URL path and protocol, user agent, client name when supplied, HTTP or WS transport, stream flag, session lineage, trace and request IDs, provider, executor, non-secret auth index, requested/alias/upstream/response models, effort, tier, status, failure classification and token detail where available. Attempts are ordered by outbound send. The final upstream body is paired with the final response. Times are RFC 3339 UTC. Latency and TTFT are integer nanoseconds. `first_byte` is the first observed HTTP response byte or WS frame. `ttft_ns` uses provider token timing where available and falls back to the first observed response byte. Optional fields can be absent for early failures or providers that omit them. +* A sealed segment ends with `type: "footer"`, `counts` and `sha256`. Hash the exact decompressed bytes of every preceding line **including its LF**. Counts are calls, blobs, unique_blob_bytes, referenced_blob_bytes and jsonl_bytes (before the footer). The dedup ratio divides bytes across all blob references by unique blob bytes. A live segment has no footer. + +The sidecar `.manifest.json` contains file, date, counts, size (sealed bytes), sha256 (sealed file), source_sha256 (same digest as footer) and optional uploaded_to. The hidden writer lock and temporary files are local implementation details. Export needs only the segment pairs, matching identities for encrypted files and any live segment to include. + +To re-inline both requests from a decrypted segment using jq alone: + +```sh +jq -nc ' + def unpack($b): + if has("body") then . as $p | .body + + (($p.parts // {}) | with_entries(.value |= + (if type == "array" then map($b[.]) else $b[.] end))) + else . end; + foreach inputs as $e ({b:{},call:null}; + .call=null | if $e.type=="blob" then .b[$e.sha256]=$e.value + elif $e.type=="call" then .b as $b | .call=($e.call | + if has("client_request") then .client_request |= unpack($b) else . end | + if has("upstream_request") then .upstream_request |= unpack($b) else . end) + else . end; + .call // empty)' restored.jsonl +``` + +## CLI and management + +```sh +t3code-proxy traces stats --dir ./traces +t3code-proxy traces verify --dir ./traces --identity trace-identity.txt +t3code-proxy traces export --dir ./traces --from 2026-11-01 --to 2026-11-20 \ + --identity trace-identity.txt --output calls.jsonl +t3code-proxy traces export --dir ./traces --identity trace-identity.txt \ + --format conversations > sessions.jsonl +``` + +Date bounds include both endpoints. Omit `--dir` to resolve it from `--config` (default `config.yaml`). Call export writes one resolved record per line. Conversation export groups records by session, putting calls without a session in individual groups. Call export holds one segment's blob dictionary in memory. Conversation export also buffers the grouped records in the selected range. Stats reads sealed manifests without a private identity. Verify checks sealed hashes, age authentication, footer and blob integrity, counts and references. Live files can only be checked through their last complete entry. Output files created with `--output` must be new paths. + +Authenticated v8 routes are `GET/PUT /v8/management/trace-archive/settings`, `GET /v8/management/trace-archive/status`, and `POST /v8/management/trace-archive/seal-now`. PUT replaces the settings block. Status includes UTC-day counts and bytes, queue depth, dropped calls, dedup ratio, local manifests and writer/upload errors. diff --git a/go.mod b/go.mod index 35a338eb4ac..a5fc6702fb3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rune-oai/t3code-proxy go 1.26.10 require ( + filippo.io/age v1.2.1 github.com/MoeForge/moego v0.1.0-alpha.3 github.com/andybalholm/brotli v1.0.6 github.com/atotto/clipboard v0.1.4 diff --git a/go.sum b/go.sum index 9e8e45ee9a3..1aa5abc4ee7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o= +filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/MoeForge/moego v0.1.0-alpha.3 h1:sn8Kg4qY/gqGXU4CCJWfgzouaim6alkvXcob0uFDohg= diff --git a/internal/api/handlers/management/config_v8.go b/internal/api/handlers/management/config_v8.go index daf368a0353..55f802a35f6 100644 --- a/internal/api/handlers/management/config_v8.go +++ b/internal/api/handlers/management/config_v8.go @@ -17,6 +17,18 @@ import ( // ConfigV8ContextKey marks operations registered by the v8 API that save configuration. const ConfigV8ContextKey = "management.config-v8" +// TraceArchiveSettings returns the default block even on a fresh installation. +func (h *Handler) TraceArchiveSettings(c *gin.Context) { + if c.Request.Method == http.MethodGet { + h.mu.Lock() + defer h.mu.Unlock() + c.JSON(http.StatusOK, h.cfg.TraceArchive) + return + } + c.Params = append(c.Params, gin.Param{Key: "path", Value: "/trace-archive"}) + h.ConfigV8(c) +} + // ConfigV8 serves the persisted configuration using v8 names. GET is a view; // only a successful configuration mutation migrates the stored document. // diff --git a/internal/api/server.go b/internal/api/server.go index 8398ed5e006..4c15aeeb652 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -17,6 +17,8 @@ import ( "sync/atomic" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/gin-gonic/gin" managementHandlers "github.com/rune-oai/t3code-proxy/internal/api/handlers/management" "github.com/rune-oai/t3code-proxy/internal/api/middleware" @@ -41,6 +43,7 @@ import ( // Server represents the main API server. // It encapsulates the Gin engine, HTTP server, handlers, and configuration. type Server struct { + archive tracearchive.Recorder // engine is the Gin web framework engine instance. engine *gin.Engine @@ -228,6 +231,8 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk engine.Use(corsMiddleware()) engine.Use(s.responseCompressionMiddleware()) + s.configureArchive(cfg) + engine.Use(s.traceMiddleware) openai.SetResponsesWebsocketCompressionEnabled(cfg.Codex.WebsocketCompressionEnabled()) s.wsAuthEnabled.Store(cfg.WebsocketAuth) s.exampleAPIKeySafeModeActive.Store(s.exampleAPIKeySafeModeRequired(cfg)) @@ -453,6 +458,7 @@ func (s *Server) Start() error { // - error: An error if the server fails to stop func (s *Server) Stop(ctx context.Context) error { log.Debug("Stopping API server...") + defer s.archive.Close() if s.keepAliveEnabled { select { diff --git a/internal/api/server_management_v8.go b/internal/api/server_management_v8.go index bbd46abe7fc..d67e362cbd2 100644 --- a/internal/api/server_management_v8.go +++ b/internal/api/server_management_v8.go @@ -37,6 +37,10 @@ func (s *Server) registerManagementV8Routes() { v8.GET("/quota-history", s.mgmt.GetQuotaHistory) v8.POST("/quota-history/refresh", s.mgmt.RefreshQuotaHistory) + v8.GET("/trace-archive/settings", s.mgmt.TraceArchiveSettings) + v8.PUT("/trace-archive/settings", s.mgmt.TraceArchiveSettings) + v8.GET("/trace-archive/status", func(c *gin.Context) { c.JSON(http.StatusOK, s.archive.Status()) }) + v8.POST("/trace-archive/seal-now", s.sealTraces) v8.GET("/observability/logs", s.mgmt.GetLogs) v8.DELETE("/observability/logs", s.mgmt.DeleteLogs) v8.GET("/observability/logs/errors", s.mgmt.GetRequestErrorLogs) diff --git a/internal/api/server_reload.go b/internal/api/server_reload.go index eaba4f4b387..977222aab1b 100644 --- a/internal/api/server_reload.go +++ b/internal/api/server_reload.go @@ -220,6 +220,7 @@ func (s *Server) UpdateClientsContext(ctx context.Context, cfg *config.Config) b s.mgmt.SetAuthManager(s.handlers.AuthManager) s.mgmt.SetPluginHost(s.pluginHost) s.mgmt.ApplyQuotaHistoryConfig(cfg) + s.configureArchive(cfg) } s.refreshPluginManagementRoutes() diff --git a/internal/api/server_trace_archive.go b/internal/api/server_trace_archive.go new file mode 100644 index 00000000000..32599dbe502 --- /dev/null +++ b/internal/api/server_trace_archive.go @@ -0,0 +1,101 @@ +package api + +import ( + "io" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/logging" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + log "github.com/sirupsen/logrus" +) + +func (s *Server) configureArchive(c *config.Config) { + if err := s.archive.Apply(c); err != nil { + log.WithError(err).Error("trace archive unavailable") + } +} + +type traceBody struct { + io.ReadCloser + c *tracearchive.Capture +} + +func (b traceBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + b.c.Client(p[:n]) + return n, err +} + +type traceWriter struct { + gin.ResponseWriter + c *tracearchive.Capture +} + +func (w traceWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } +func (w traceWriter) Write(p []byte) (int, error) { + n, e := w.ResponseWriter.Write(p) + w.c.ClientResponse(p[:n]) + return n, e +} +func (w traceWriter) WriteString(p string) (int, error) { + n, e := w.ResponseWriter.WriteString(p) + w.c.ClientResponse([]byte(p[:n])) + return n, e +} + +func (s *Server) traceMiddleware(c *gin.Context) { + if !s.archive.Enabled() && !(c.Request.Method == http.MethodGet && strings.HasSuffix(c.FullPath(), "/responses")) { + return + } + path := c.FullPath() + var protocol string + switch { + case strings.Contains(path, "/v1/messages"): + protocol = "claude-messages" + case strings.HasSuffix(path, "/responses") || strings.HasSuffix(path, "/responses/compact"): + protocol = "openai-responses" + case strings.HasSuffix(path, "/alpha/search"): + protocol = "codex-search" + case path == "/muse-code/search" || path == "/muse-code/browser_open": + protocol = "muse-code" + case strings.HasSuffix(path, "/chat/completions"): + protocol = "chat-completions" + case strings.Contains(path, "/v1beta/models/") || strings.HasSuffix(path, "/interactions"): + protocol = "gemini" + case strings.HasSuffix(path, "/completions") || strings.Contains(path, "/images/") || strings.Contains(path, "/videos") || strings.HasSuffix(path, "/audio/speech") || strings.HasSuffix(path, "/tts"): + protocol = "openai" + default: + return + } + if c.Request.Method != http.MethodPost && !(protocol == "openai-responses" && c.Request.Method == http.MethodGet) { + return + } + ctx := s.archive.Bind(c.Request, protocol, logging.GetGinRequestID(c)) + if c.Request.Method == http.MethodGet { + c.Request = c.Request.WithContext(ctx) + return + } + ctx, capture := tracearchive.Begin(ctx, false, nil) + if capture == nil { + return + } + c.Request = c.Request.WithContext(ctx) + if c.Request.Body != nil { + c.Request.Body = traceBody{c.Request.Body, capture} + } + w := traceWriter{c.Writer, capture} + c.Writer = w + defer func() { capture.Finish(w.Status()) }() + c.Next() +} + +func (s *Server) sealTraces(c *gin.Context) { + if err := s.archive.Seal(c.Request.Context()); err != nil { + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"status": "ok"}) +} diff --git a/internal/api/server_trace_archive_test.go b/internal/api/server_trace_archive_test.go new file mode 100644 index 00000000000..76a7e4991e6 --- /dev/null +++ b/internal/api/server_trace_archive_test.go @@ -0,0 +1,236 @@ +package api + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "testing" + + "github.com/MoeForge/moego/ojson" + "github.com/gorilla/websocket" + "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/registry" + "github.com/rune-oai/t3code-proxy/internal/runtime/executor" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" +) + +func TestTraceArchiveStreamingClaudeThroughServer(t *testing.T) { + const model = "archive-claude-test" + const body = `{"model":"archive-claude-test","max_tokens":64,"messages":[{"role":"user","content":"Check the sky"}],"stream":true}` + wire := make(chan []byte, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + wire <- b + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Set-Cookie", "never-archive-cookie") + for _, e := range []string{ + `{"type":"message_start","message":{"type":"message","id":"msg_demo","model":"archive-claude-test","role":"assistant","content":[],"usage":{"input_tokens":8}}}`, + `{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}`, + `{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Check the forecast."}}`, + `{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"demo-signature"}}`, + `{"type":"content_block_stop","index":0}`, + `{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"tool_1","name":"forecast","input":{}}}`, + `{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\"Seoul\"}"}}`, + `{"type":"content_block_stop","index":1}`, + `{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":12}}`, + `{"type":"message_stop"}`, + } { + fmt.Fprintf(w, "event: %s\ndata: %s\n\n", ojson.Get(e, "type").String(), e) + w.(http.Flusher).Flush() + } + })) + defer upstream.Close() + cfg := &config.Config{TraceArchive: config.TraceArchiveConfig{Enabled: true, Dir: t.TempDir()}} + cfg.APIKeys = []string{"never-archive-client-secret"} + s := newTestServerWithConfig(t, cfg) + defer s.archive.Close() + manager := s.handlers.AuthManager + manager.RegisterExecutor(executor.NewClaudeExecutor(cfg)) + credential := &auth.Auth{ID: "trace-e2e", Provider: "claude", Status: auth.StatusActive, Attributes: map[string]string{"api_key": "never-archive-upstream-secret", "base_url": upstream.URL}} + if _, err := manager.Register(context.Background(), credential); err != nil { + t.Fatal(err) + } + models := registry.GetGlobalRegistry() + models.RegisterClient(credential.ID, "claude", []*registry.ModelInfo{{ID: model}}) + defer models.UnregisterClient(credential.ID) + req := httptest.NewRequest("POST", "/v1/messages", strings.NewReader(body)) + req.Header.Set("X-Api-Key", "never-archive-client-secret") + req.Header.Set("Authorization", "Bearer never-archive-client-secret") + req.Header.Set("Cookie", "never-archive-cookie") + req.Header.Set("Content-Type", "application/json") + rr := httptest.NewRecorder() + s.Handler().ServeHTTP(rr, req) + if rr.Code != 200 || !strings.Contains(rr.Body.String(), "message_stop") { + t.Fatalf("status=%d body=%s", rr.Code, rr.Body) + } + if err := s.archive.Seal(context.Background()); err != nil { + t.Fatal(err) + } + var out, stderr bytes.Buffer + if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir}, &out, &stderr) != 0 { + t.Fatal(stderr.String()) + } + result := ojson.ParseBytes(out.Bytes()) + for k, want := range map[string]string{"transport": "http", "client_protocol": "claude-messages", "requested_model": model, "response.content.0.signature": "demo-signature", "response.content.0.thinking": "Check the forecast.", "response.content.1.input.city": "Seoul", "response.stop_reason": "tool_use", "token_usage.OutputTokens": "12", "attempts.0.status": "200"} { + if result.Get(k).String() != want { + t.Errorf("%s = %q; want %q", k, result.Get(k).String(), want) + } + } + if bytes.Contains(out.Bytes(), []byte("never-archive")) { + t.Fatal("transport credential in archive") + } + for key, want := range map[string][]byte{"client_request": []byte(body), "upstream_request": <-wire} { + if !reflect.DeepEqual(result.Get(key).Value(), ojson.ParseBytes(want).Value()) { + t.Fatalf("%s differs from wire JSON", key) + } + } + t.Logf("exported response: %s", result.Get("response").Raw) +} + +func TestTraceArchiveCodexWebsocketThroughServer(t *testing.T) { + const model = "trace-ws-model" + const original = `{"type":"response.create","model":"trace-ws-model","input":[{"role":"user","content":"hello"}]}` + const done = `{"type":"response.output_item.done","output_index":0,"item":{"id":"i1","type":"reasoning","encrypted_content":"sealed-reasoning","summary":[{"type":"summary_text","text":"abbreviated"}]}}` + const completed = `{"type":"response.completed","response":{"id":"ws-record","model":"trace-ws-model","output":[],"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}` + sent := make(chan []byte, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var upgrader websocket.Upgrader + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer func() { + if errClose := conn.Close(); errClose != nil { + t.Log(errClose) + } + }() + _, p, err := conn.ReadMessage() + if err != nil { + t.Error(err) + return + } + sent <- p + for _, frame := range []string{`{"type":"response.created","response":{"id":"ws-record","output":[]}}`, done, completed} { + if err = conn.WriteMessage(websocket.TextMessage, []byte(frame)); err != nil { + t.Error(err) + return + } + } + // Keep the connection available until the downstream client completes. + _, _, _ = conn.ReadMessage() + })) + defer upstream.Close() + cfg := &config.Config{TraceArchive: config.TraceArchiveConfig{Enabled: true, Dir: t.TempDir()}} + cfg.APIKeys = []string{"test-key"} + s := newTestServerWithConfig(t, cfg) + defer s.archive.Close() + manager := s.handlers.AuthManager + manager.RegisterExecutor(executor.NewCodexAutoExecutor(cfg)) + credential := &auth.Auth{ID: "trace-ws-e2e", Provider: "codex", Status: auth.StatusActive, Attributes: map[string]string{"websockets": "true", "api_key": "fake-ws-key", "base_url": upstream.URL}} + if _, err := manager.Register(context.Background(), credential); err != nil { + t.Fatal(err) + } + catalog := registry.GetGlobalRegistry() + catalog.RegisterClient(credential.ID, "codex", []*registry.ModelInfo{{ID: model}}) + defer catalog.UnregisterClient(credential.ID) + finished := make(chan struct{}) + hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer close(finished); s.Handler().ServeHTTP(w, r) })) + defer hub.Close() + conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(hub.URL, "http")+"/v1/responses", http.Header{"Authorization": {"Bearer test-key"}}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + if err = conn.WriteMessage(websocket.TextMessage, []byte(original)); err != nil { + t.Fatal(err) + } + for { + _, frame, errRead := conn.ReadMessage() + if errRead != nil { + t.Fatal(errRead) + } + typ := ojson.GetBytes(frame, "type").String() + if typ == "error" || typ == "response.failed" { + t.Fatal(string(frame)) + } + if typ == "response.completed" { + break + } + } + if err = conn.Close(); err != nil { + t.Fatal(err) + } + <-finished + if err = s.archive.Seal(context.Background()); err != nil { + t.Fatal(err) + } + var out, stderr bytes.Buffer + if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir}, &out, &stderr) != 0 { + t.Fatal(stderr.String()) + } + record := ojson.ParseBytes(out.Bytes()) + if record.Get("response.output.0.encrypted_content").String() != "sealed-reasoning" || record.Get("response.output.0.summary.0.text").String() != "abbreviated" || record.Get("transport").String() != "ws" || record.Get("attempts.0.transport").String() != "ws" { + t.Fatal(out.String()) + } + if !reflect.DeepEqual(record.Get("client_request").Value(), ojson.Parse(original).Value()) || !reflect.DeepEqual(record.Get("upstream_request").Value(), ojson.ParseBytes(<-sent).Value()) { + t.Fatal("websocket payload differs from wire") + } +} + +func TestTraceArchiveManagement(t *testing.T) { + t.Setenv("MANAGEMENT_PASSWORD", "test-password") + cfg := &config.Config{TraceArchive: config.TraceArchiveConfig{Dir: t.TempDir()}} + s := newTestServerWithConfig(t, cfg, WithLocalManagementPassword("test-password")) + defer s.archive.Close() + if err := os.WriteFile(s.configFilePath, []byte("config-version: 8\n"), 0600); err != nil { + t.Fatal(err) + } + request := func(method, path, body string, status int) string { + t.Helper() + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.RemoteAddr = "127.0.0.1:1234" + req.Header.Set("Authorization", "Bearer test-password") + w := httptest.NewRecorder() + s.engine.ServeHTTP(w, req) + if w.Code != status { + t.Fatalf("%s %s: %d %s", method, path, w.Code, w.Body.String()) + } + return w.Body.String() + } + const base = "/v8/management/trace-archive/" + request("GET", "/v0/management/trace-archive/status", "", 404) + request("GET", base+"settings", "", 200) + request("POST", base+"seal-now", "", 409) + request("PUT", base+"settings", `{"enabled":true,"recipients":["bad"]}`, 422) + settings := ojson.NewDocString(`{"enabled":true}`) + _ = settings.Set("dir", cfg.TraceArchive.Dir) + request("PUT", base+"settings", settings.String(), 200) + if !ojson.Get(request("GET", base+"settings", "", 200), "enabled").Bool() { + t.Fatal("settings read missed persisted update") + } + // Exercise the same update callback as the file watcher. + raw, err := os.ReadFile(s.configFilePath) + if err != nil { + t.Fatal(err) + } + next, err := config.ParseConfigBytes(raw) + if err != nil { + t.Fatal(err) + } + s.UpdateClients(next) + if !ojson.Get(request("GET", base+"status", "", 200), "enabled").Bool() { + t.Fatal("watcher did not enable recorder") + } + request("POST", base+"seal-now", "", 200) +} diff --git a/internal/config/config.go b/internal/config/config.go index a6c26157110..b911c48510a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ type ServerResponseCompression = ResponseCompressionConfig // Config represents the application's configuration, loaded from a YAML file. type Config struct { + TraceArchive TraceArchiveConfig `yaml:"trace-archive,omitempty" json:"trace-archive,omitempty"` // ServerResponseCompression is the JSON-visible mirror of the downstream // response compression settings (YAML: server.response-compression). ServerResponseCompression ResponseCompressionConfig `yaml:"-" json:"response-compression"` diff --git a/internal/config/config_load.go b/internal/config/config_load.go index 70214776779..d90bff89c5d 100644 --- a/internal/config/config_load.go +++ b/internal/config/config_load.go @@ -242,6 +242,9 @@ func LoadConfigOptional(configFile string, optional bool) (*Config, error) { if errValidate := cfg.ValidateCredentialWeights(); errValidate != nil { return nil, errValidate } + if errValidate := cfg.TraceArchive.Validate(); errValidate != nil { + return nil, errValidate + } if errValidate := cfg.ValidatePlanRouting(); errValidate != nil { return nil, errValidate } diff --git a/internal/config/config_v8.go b/internal/config/config_v8.go index 399456ff7e1..09b84eadc21 100644 --- a/internal/config/config_v8.go +++ b/internal/config/config_v8.go @@ -635,7 +635,7 @@ func IsV8ConfigLayout(root *yaml.Node) bool { func v8AllowedRoots() map[string]bool { allowed := map[string]bool{"models": true, "config-version": true, "api-keys": true, "plugins": true, "quota-exceeded": true, "client": true, - "quota-history": true} // quota-history: t3code fork recorder section. + "quota-history": true, "trace-archive": true} for _, path := range v8Paths { section, _, _ := strings.Cut(path.current, ".") allowed[section] = true diff --git a/internal/config/parse.go b/internal/config/parse.go index c20f2dd17a9..243d9b56ae4 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -54,6 +54,9 @@ func ParseConfigBytes(data []byte) (*Config, error) { if errValidate := cfg.ValidateCredentialWeights(); errValidate != nil { return nil, errValidate } + if errValidate := cfg.TraceArchive.Validate(); errValidate != nil { + return nil, errValidate + } if errValidate := cfg.ValidatePlanRouting(); errValidate != nil { return nil, errValidate } diff --git a/internal/config/trace_archive.go b/internal/config/trace_archive.go new file mode 100644 index 00000000000..847f63823cd --- /dev/null +++ b/internal/config/trace_archive.go @@ -0,0 +1,40 @@ +package config + +import ( + "fmt" + "strings" + + "filippo.io/age" +) + +type TraceArchiveConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + Dir string `yaml:"dir,omitempty" json:"dir"` + Recipients []string `yaml:"recipients,omitempty" json:"recipients"` + QueueSize int `yaml:"queue-size,omitempty" json:"queue-size"` + MaxLocalSizeMB int64 `yaml:"max-local-size-mb,omitempty" json:"max-local-size-mb"` + ExcludeModels []string `yaml:"exclude-models,omitempty" json:"exclude-models"` + ExcludeEndpoints []string `yaml:"exclude-endpoints,omitempty" json:"exclude-endpoints"` + Upload TraceUploadConfig `yaml:"upload,omitempty" json:"upload"` +} + +type TraceUploadConfig struct { + Enabled bool `yaml:"enabled" json:"enabled"` + Binary string `yaml:"binary,omitempty" json:"binary"` + Remote string `yaml:"remote,omitempty" json:"remote"` +} + +func (c TraceArchiveConfig) Validate() error { + if c.QueueSize < 0 || c.QueueSize > 65536 || c.MaxLocalSizeMB < 0 || c.MaxLocalSizeMB > 1<<40 { + return fmt.Errorf("trace-archive: invalid queue-size or max-local-size-mb") + } + for _, key := range c.Recipients { + if _, err := age.ParseX25519Recipient(key); err != nil { + return fmt.Errorf("trace-archive: invalid X25519 recipient") + } + } + if c.Upload.Enabled && (strings.TrimSpace(c.Upload.Remote) == "" || strings.HasPrefix(c.Upload.Remote, "-")) { + return fmt.Errorf("trace-archive: upload.remote is required") + } + return nil +} diff --git a/internal/doctor/config.go b/internal/doctor/config.go index fdc9e797d8c..c0ab396a75f 100644 --- a/internal/doctor/config.go +++ b/internal/doctor/config.go @@ -16,6 +16,10 @@ import ( // decoded directly with yaml.v3 so the doctor does not depend on the server's // config package. type hubConfig struct { + TraceArchive struct { + Enabled bool `yaml:"enabled"` + Recipients []string `yaml:"recipients"` + } `yaml:"trace-archive"` Host string `yaml:"host"` Port int `yaml:"port"` Keys []string `yaml:"api-keys"` @@ -99,6 +103,9 @@ func checkConfig(e *env) Result { c := e.cfg host := c.Server.Host var problems, warnings []string + if c.TraceArchive.Enabled && len(c.TraceArchive.Recipients) == 0 { + warnings = append(warnings, "trace archive has no age recipients; sealed history is plaintext") + } switch host { case "127.0.0.1", "localhost", "::1": case "": diff --git a/internal/runtime/executor/codex_websockets_duplex.go b/internal/runtime/executor/codex_websockets_duplex.go index fc2523cd7ef..00d7a27df00 100644 --- a/internal/runtime/executor/codex_websockets_duplex.go +++ b/internal/runtime/executor/codex_websockets_duplex.go @@ -11,6 +11,8 @@ import ( "strings" "sync" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/runtime/executor/helps" @@ -66,6 +68,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // Explicit creates have their own request settings. Automatic successors // inherit the preceding response settings, just as they do upstream. var metadataMu sync.Mutex + initial.archive = tracearchive.From(ctx) pending := []*codexWebsocketPrepared{initial} // Serialize explicit creates against steering continuations. A response.created // alone does not identify whether upstream created it automatically. @@ -109,6 +112,9 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( return } } + if p := steeringSettings[parent]; p != nil && p.archive != current.archive { + p.archive.Finish(http.StatusAccepted) + } delete(steeringSettings, parent) } removeAcceptedSteer := func(id string) { @@ -625,6 +631,8 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( fail(fmt.Errorf("too many outstanding response.create requests")) return false } + turnCtx, capture := tracearchive.Begin(streamCtx, true, consumedPayload) + prepared.archive = capture pending = append(pending, prepared) consumedPayload = nil // pending owns it now, including a possibly partial write metadataMu.Unlock() @@ -635,7 +643,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( fail(fmt.Errorf("websocket credential is no longer enabled")) return false } - helps.RecordAPIWebsocketRequest(streamCtx, e.cfg, helps.UpstreamRequestLog{ + helps.RecordAPIWebsocketRequest(capture.Context(turnCtx), e.cfg, helps.UpstreamRequestLog{ URL: initial.wsURL, Method: "WEBSOCKET", Body: payload, Provider: e.Identifier(), AuthID: auth.ID, }) if errWrite := writeCodexWebsocketMessage(sess, conn, payload); errWrite != nil { @@ -756,8 +764,16 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( metadataMu.Lock() unacknowledgedSteers = append(unacknowledgedSteers, codexSteerSubmission{parent: parent, eventID: steerEventID}) consumedPayload = nil + steerCtx, steerArchive := tracearchive.Begin(streamCtx, true, message.Payload) if settings != nil { - steeringSettings[parent] = settings + snapshot := *settings + snapshot.archive = steerArchive + if prior := steeringSettings[parent]; prior != nil { + prior.archive.Finish(http.StatusAccepted) + } + steeringSettings[parent] = &snapshot + } else { + steerArchive.Finish(http.StatusAccepted) } metadataMu.Unlock() // Control frames bypass response.create translations and built-in defaults. @@ -767,7 +783,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( fail(fmt.Errorf("websocket credential is no longer enabled")) return } - helps.RecordAPIWebsocketRequest(streamCtx, e.cfg, helps.UpstreamRequestLog{ + helps.RecordAPIWebsocketRequest(steerArchive.Context(steerCtx), e.cfg, helps.UpstreamRequestLog{ URL: initial.wsURL, Method: "WEBSOCKET", Body: payload, Provider: e.Identifier(), AuthID: auth.ID, }) if errWrite := writeCodexWebsocketMessage(sess, conn, payload); errWrite != nil { @@ -809,6 +825,13 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // releasing the execution session. No goroutine outlives its socket. e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "duplex_closed", nil) <-writerDone + current.archive.Finish(http.StatusBadGateway) + for _, p := range pending { + p.archive.Finish(http.StatusBadGateway) + } + for _, p := range steeringSettings { + p.archive.Finish(http.StatusBadGateway) + } sess.clearActive(conn, readCh) unlock() }() @@ -951,6 +974,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // Retain response settings, not request history or authorization headers. // In-flight steering pins its parent's settings independently of this window. snapshot := *current + snapshot.archive = nil snapshot.originalPayload, snapshot.wsHeaders = nil, nil snapshotDoc := ojson.NewDoc([]byte("{}")) if reasoning := ojson.GetBytesNoCopy(current.clientBody, "reasoning"); reasoning.Exists() { @@ -979,7 +1003,8 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( outputFallback = nil } observeCodexTokenEvent(reporter, payload) - helps.AppendCodexAPIWebsocketResponse(ctx, e.cfg, payload) + frameCtx := current.archive.Context(ctx) + helps.AppendCodexAPIWebsocketResponse(frameCtx, e.cfg, payload) helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload) // Steering acknowledgements, pending notifications and failures are opaque: // preserve their IDs, input, sequence numbers and event types byte-for-byte. @@ -1102,7 +1127,11 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( var status interface{ StatusCode() int } terminalBusinessRejection = errors.As(terminalErr, &status) && status.StatusCode() == http.StatusBadRequest helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", terminalErr) - eventReporter.PublishFailure(ctx, terminalErr) + eventReporter.PublishFailure(eventPrepared.archive.Context(ctx), terminalErr) + if eventPrepared != current { + eventPrepared.archive.Chunk(payload) + } + eventPrepared.archive.Finish(http.StatusBadGateway) if isInitialResponse { send(cliproxyexecutor.StreamChunk{Err: terminalErr}) return @@ -1144,10 +1173,11 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( cacheCodexReasoningReplayFromCompleted(current.replayScope, payload) } if detail, ok := helps.ParseCodexUsageWithOutputItems(payload, outputItems, outputFallback); ok { - reporter.Publish(ctx, detail) + reporter.Publish(frameCtx, detail) } else { - reporter.EnsurePublished(ctx) + reporter.EnsurePublished(frameCtx) } + current.archive.Finish(http.StatusOK) publishCodexImageToolUsageWithOutputItems(ctx, reporter, current.clientBody, payload, outputItems, outputFallback) if e.duplexStateObserver != nil { e.duplexStateObserver("terminal_pending", false) diff --git a/internal/runtime/executor/codex_websockets_stream.go b/internal/runtime/executor/codex_websockets_stream.go index 92a1ebaf5ef..133e874fcb9 100644 --- a/internal/runtime/executor/codex_websockets_stream.go +++ b/internal/runtime/executor/codex_websockets_stream.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/config" @@ -861,6 +863,7 @@ func (e *CodexWebsocketsExecutor) ExecuteStream(ctx context.Context, auth *clipr // codexWebsocketPrepared contains the request pipeline output shared by the // ordinary per-response stream and subsequent creates on a duplex connection. type codexWebsocketPrepared struct { + archive *tracearchive.Capture from sdktranslator.Format responseFormat sdktranslator.Format to sdktranslator.Format diff --git a/internal/runtime/executor/helps/logging_helpers.go b/internal/runtime/executor/helps/logging_helpers.go index 0adbaa5df85..0ec205ff027 100644 --- a/internal/runtime/executor/helps/logging_helpers.go +++ b/internal/runtime/executor/helps/logging_helpers.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gin-gonic/gin" "github.com/rune-oai/t3code-proxy/internal/config" @@ -65,6 +67,7 @@ func requestLogCaptureEnabled(cfg *config.Config) bool { // RecordAPIRequest stores the upstream request metadata in Gin context for request logging. func RecordAPIRequest(ctx context.Context, cfg *config.Config, info UpstreamRequestLog) { + tracearchive.From(ctx).Request(info.Provider, "http", info.Body) if cfg == nil || cfg.CommercialMode { return } @@ -291,6 +294,7 @@ func newAPIRequestLogBuilder(index int, info UpstreamRequestLog, timestamp time. // RecordAPIResponseMetadata captures upstream response status/header information for the latest attempt. func RecordAPIResponseMetadata(ctx context.Context, cfg *config.Config, status int, headers http.Header) { + tracearchive.From(ctx).Status(status) logging.SetResponseHeaders(ctx, headers) if !requestLogCaptureEnabled(cfg) { recordDeferredResponseMetadata(ctx, cfg, status, headers) @@ -350,6 +354,7 @@ func RecordAPIResponseError(ctx context.Context, cfg *config.Config, err error) // AppendAPIResponseChunk appends an upstream response chunk to Gin context for request logging. func AppendAPIResponseChunk(ctx context.Context, cfg *config.Config, chunk []byte) { + tracearchive.From(ctx).Chunk(chunk) if !requestLogCaptureEnabled(cfg) { appendDeferredResponseChunk(ctx, cfg, chunk) return @@ -395,6 +400,7 @@ func AppendAPIResponseChunk(ctx context.Context, cfg *config.Config, chunk []byt // RecordAPIWebsocketRequest stores an upstream websocket request event in Gin context. func RecordAPIWebsocketRequest(ctx context.Context, cfg *config.Config, info UpstreamRequestLog) { + tracearchive.From(ctx).Request(info.Provider, "ws", info.Body) if cfg == nil || cfg.CommercialMode { return } @@ -431,6 +437,7 @@ func RecordAPIWebsocketRequest(ctx context.Context, cfg *config.Config, info Ups // RecordAPIWebsocketHandshake stores the upstream websocket handshake response metadata. func RecordAPIWebsocketHandshake(ctx context.Context, cfg *config.Config, status int, headers http.Header) { + tracearchive.From(ctx).Status(status) logging.SetResponseHeaders(ctx, headers) if !requestLogCaptureEnabled(cfg) { recordDeferredResponseMetadata(ctx, cfg, status, headers) @@ -501,6 +508,7 @@ func WebsocketUpgradeRequestURL(rawURL string) string { // AppendAPIWebsocketResponse stores an upstream websocket response frame in Gin context. func AppendAPIWebsocketResponse(ctx context.Context, cfg *config.Config, payload []byte) { + tracearchive.From(ctx).Chunk(payload) if !requestLogCaptureEnabled(cfg) { appendDeferredResponseChunk(ctx, cfg, payload) return diff --git a/internal/runtime/executor/helps/usage_helpers.go b/internal/runtime/executor/helps/usage_helpers.go index d44067c224d..5f7cd8e41c3 100644 --- a/internal/runtime/executor/helps/usage_helpers.go +++ b/internal/runtime/executor/helps/usage_helpers.go @@ -9,12 +9,15 @@ import ( "mime" "mime/multipart" "net/http" + "net/http/httptrace" "reflect" "strings" "sync" "sync/atomic" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -836,6 +839,9 @@ type usageTTFTRoundTripper struct { func (t usageTTFTRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { cliproxyexecutor.MarkUpstreamAttempt(req.Context()) t.reporter.StartResponseTTFT() + if capture := tracearchive.From(req.Context()); capture != nil { + req = req.WithContext(httptrace.WithClientTrace(req.Context(), &httptrace.ClientTrace{GotFirstResponseByte: capture.FirstByte})) + } resp, errRoundTrip := t.base.RoundTrip(req) if errRoundTrip != nil { return resp, errRoundTrip diff --git a/internal/tracearchive/archive_test.go b/internal/tracearchive/archive_test.go new file mode 100644 index 00000000000..04ba63aceb0 --- /dev/null +++ b/internal/tracearchive/archive_test.go @@ -0,0 +1,281 @@ +package tracearchive + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "filippo.io/age" + "github.com/MoeForge/moego/ojson" + "github.com/rune-oai/t3code-proxy/internal/config" +) + +func check(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} +func recorder(t *testing.T, settings config.TraceArchiveConfig) *Recorder { + t.Helper() + settings.Enabled = true + r := new(Recorder) + check(t, r.Apply(&config.Config{TraceArchive: settings})) + t.Cleanup(r.Close) + return r +} +func submit(t *testing.T, r *Recorder, body, response string) { + t.Helper() + req := httptest.NewRequest("POST", "/v1/messages", nil) + ctx := r.Bind(req, "claude-messages", "trace") + _, c := Begin(ctx, false, []byte(body)) + if c == nil { + t.Fatal("dropped test call") + } + c.Request("claude", "http", []byte(body)) + c.Chunk([]byte(response)) + c.Finish(200) +} +func TestGrowingConversationAgeExport(t *testing.T) { + dir := t.TempDir() + id, err := age.GenerateX25519Identity() + check(t, err) + r := recorder(t, config.TraceArchiveConfig{Dir: dir, Recipients: []string{id.Recipient().String()}}) + var msgs []string + var bodies []string + resp := `{"content":[{"type":"thinking","thinking":"h","signature":"proof"},{"type":"redacted_thinking","data":"opaque"}],"stop_reason":"end_turn"}` + for i := range 12 { + msgs = append(msgs, fmt.Sprintf(`{"role":"user","content":"%s%d"}`, strings.Repeat("abc", 200), i)) + b := fmt.Sprintf(`{"system":"shared","tools":[{"name":"f","n":9007199254740993}],"messages":[%s]}`, strings.Join(msgs, ",")) + bodies = append(bodies, b) + submit(t, r, b, resp) + } + check(t, r.Seal(context.Background())) + ms, err := listManifests(dir) + check(t, err) + if len(ms) != 1 || ms[0].Counts.Blobs != 14 || ms[0].Counts.Calls != 12 { + t.Fatalf("%+v", ms) + } + c := ms[0].Counts + t.Logf("%d referenced bytes / %d unique bytes = %.2fx", c.Referenced, c.Unique, c.ratio()) + if c.ratio() < 10 { + t.Fatal(c) + } + key := filepath.Join(t.TempDir(), "key") + check(t, os.WriteFile(key, []byte(id.String()+"\n"), 0600)) + for _, mode := range []string{"verify", "export"} { + var out bytes.Buffer + err = command([]string{mode, "--dir", dir, "--identity", key}, &out, io.Discard) + check(t, err) + if mode == "export" { + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != len(bodies) { + t.Fatal(len(lines)) + } + for i, line := range lines { + for k, v := range map[string]string{"client_request": bodies[i], "upstream_request": bodies[i], "response": resp} { + if !bytes.Equal(canonical(nil, ojson.Get(line, k)), canonical(nil, ojson.Parse(v))) { + t.Fatalf("round trip changed %s in turn %d", k, i) + } + } + } + } + } +} +func TestInvalidFooterAndUnresolvedCall(t *testing.T) { + dir := t.TempDir() + s, err := newSegment(dir, today()) + check(t, err) + check(t, s.append(&call{Metadata: Metadata{SchemaVersion: 1}, client: []byte(`{"messages":["hi"]}`)})) + check(t, s.f.Close()) + raw, err := os.ReadFile(s.path) + check(t, err) + footer := entry("footer") + _ = footer.Set("counts", s.counts) + _ = footer.Set("sha256", strings.Repeat("0", 64)) + if _, err = scan(bytes.NewReader(append(raw, append(footer.Bytes(), '\n')...)), s.date, true, nil); err == nil { + t.Fatal("accepted bad footer") + } + changed := bytes.ReplaceAll(raw, []byte(`"messages":["`), []byte(`"messages":["bad-`)) + if _, err = scan(bytes.NewReader(changed), s.date, false, nil); err == nil { + t.Fatal("accepted broken reference") + } +} + +func TestRawFallbackAndFilters(t *testing.T) { + dir := t.TempDir() + r := recorder(t, config.TraceArchiveConfig{Dir: dir, ExcludeModels: []string{"private"}, ExcludeEndpoints: []string{"/excluded"}}) + req := httptest.NewRequest("POST", "/excluded", nil) + if _, c := Begin(r.Bind(req, "messages", "trace"), false, nil); c != nil { + t.Fatal("captured excluded endpoint") + } + submit(t, r, `{"model":"private"}`, `{"private":"never-store"}`) + submit(t, r, `{"model":"public"}`, "data: truncated") + r.Close() + var out bytes.Buffer + check(t, readArchive(dir, "", "", "export", "calls", nil, &out)) + if strings.Count(out.String(), "\n") != 1 || ojson.GetBytes(out.Bytes(), "raw_frames_base64.0").String() != "ZGF0YTogdHJ1bmNhdGVk" || bytes.Contains(out.Bytes(), []byte("private")) { + t.Fatal(out.String()) + } +} + +func TestTornTailAndStaleSeal(t *testing.T) { + dir := t.TempDir() + s, err := newSegment(dir, "2000-01-01") + check(t, err) + check(t, s.append(&call{Metadata: Metadata{SchemaVersion: 1}, client: []byte(`{"input":["old"]}`)})) + _, err = s.f.WriteString(`{"type":"blo`) + check(t, err) + check(t, s.f.Close()) + r := recorder(t, config.TraceArchiveConfig{Dir: dir}) + submit(t, r, `{"input":["old"]}`, `{"ok":true}`) + r.Close() + var b bytes.Buffer + check(t, command([]string{"verify", "--dir", dir}, &b, io.Discard)) + if strings.Count(b.String(), "1 calls, 1 blobs") != 2 { + t.Fatal(b.String()) + } + paths, err := filepath.Glob(filepath.Join(dir, "*.jsonl.zst")) + check(t, err) + f, err := os.OpenFile(paths[0], os.O_APPEND|os.O_WRONLY, 0600) + check(t, err) + _, err = f.WriteString("x") + check(t, err) + check(t, f.Close()) + if command([]string{"verify", "--dir", dir}, io.Discard, io.Discard) == nil { + t.Fatal("corrupt sealed file accepted") + } +} +func TestQueueAdmissionAndHeaderMask(t *testing.T) { + dir := t.TempDir() + r := recorder(t, config.TraceArchiveConfig{Dir: dir, QueueSize: 1}) + req := httptest.NewRequest("POST", "/v1/responses", nil) + for _, h := range []string{"Authorization", "X-api-key", "Cookie"} { + req.Header.Set(h, h+"-secret") + } + ctx := r.Bind(req, "responses", "trace") + body := []byte(`{"headers":{"AUTHORIZATION":"a-secret","X-API-Key":"x-secret","Cookie":"c-secret"},"response_headers":[{"name":"Set-Cookie","value":"named-secret"},["Authorization","tuple-secret"],"x-api-key: string-secret"],"input":"hi"}`) + _, first := Begin(ctx, true, body) + for range 1000 { + if _, c := Begin(ctx, true, body); c != nil { + t.Fatal("admitted a turn beyond capacity") + } + } + first.Request("codex", "ws", body) + first.Chunk(body) + first.Finish(200) + r.Close() + if r.dropped.Load() != 1000 { + t.Fatal(r.dropped.Load()) + } + paths, err := filepath.Glob(filepath.Join(dir, "*.live")) + check(t, err) + b, err := os.ReadFile(paths[0]) + check(t, err) + if bytes.Contains(b, []byte("-secret")) || !bytes.Contains(b, []byte("[masked]")) { + t.Fatal("failed to mask header") + } + if !bytes.Contains(body, []byte("a-secret")) { + t.Fatal("mutated caller payload") + } +} +func TestReconstructProviders(t *testing.T) { + claude := []string{ + `{"type":"message_start","message":{"model":"claude","content":[],"usage":{"input_tokens":17}}}`, + `{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}`, + `{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"plan"}}`, + `{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"signed"}}`, + `{"type":"content_block_stop","index":0}`, + `{"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"f1","input":{}}}`, + `{"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"n\":42}"}}`, + `{"type":"content_block_stop","index":1}`, + `{"type":"content_block_start","index":2,"content_block":{"type":"redacted_thinking","data":"opaque"}}`, + `{"type":"content_block_stop","index":2}`, + `{"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":8}}`, + `{"type":"message_stop"}`, + } + responses := []string{`{"type":"response.output_item.done","output_index":0,"item":{"type":"reasoning","encrypted_content":"cipher","summary":[{"text":"summary"}]}}`, `{"type":"response.completed","response":{"id":"r","output":[]}}`} + chat := []string{`{"choices":[{"index":0,"delta":{"role":"assistant","content":"hel","tool_calls":[{"index":0,"id":"t","type":"function","function":{"name":"f","arguments":"{"}}]}}]}`, `{"choices":[{"index":0,"delta":{"content":"lo","tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"finish_reason":"tool_calls"}]}`, `[DONE]`} + for _, tt := range []struct { + name string + events []string + fields map[string]string + }{ + {"claude", claude, map[string]string{"content.0.thinking": "plan", "content.0.signature": "signed", "content.1.input.n": "42", "content.2.data": "opaque", "stop_reason": "tool_use", "usage.input_tokens": "17"}}, + {"responses", responses, map[string]string{"output.0.encrypted_content": "cipher", "output.0.summary.0.text": "summary"}}, + {"codex-ws", responses, map[string]string{"output.0.encrypted_content": "cipher", "id": "r"}}, + {"chat", chat, map[string]string{"choices.0.message.content": "hello", "choices.0.message.tool_calls.0.function.arguments": "{}", "choices.0.finish_reason": "tool_calls"}}, + } { + t.Run(tt.name, func(t *testing.T) { + var chunks [][]byte + for _, e := range tt.events { + if tt.name != "codex-ws" { + e = "data: " + e + } + chunks = append(chunks, []byte(e)) + } + got, ok := reconstruct(chunks) + if !ok { + t.Fatal("reconstruction failed") + } + for k, v := range tt.fields { + if ojson.GetBytes(got, k).String() != v { + t.Fatalf("%s missing from %s", k, got) + } + } + if _, ok = reconstruct(chunks[:len(chunks)-1]); ok { + t.Fatal("incomplete stream discarded") + } + }) + } +} +func TestUploadRetryAndRetention(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses POSIX fake rclone") + } + dir := t.TempDir() + remote := t.TempDir() + fake := filepath.Join(t.TempDir(), "rclone") + check(t, os.WriteFile(fake, []byte("#!/bin/sh\nset -eu\ncase $1 in\ncopyto) cp \"$3\" \"$4\";;\ncat) if [ -e \"$FAIL_RCLONE\" ]; then echo corrupt; else cat \"$3\"; fi;;\nesac\n"), 0700)) + flag := filepath.Join(t.TempDir(), "fail") + t.Setenv("FAIL_RCLONE", flag) + check(t, os.WriteFile(flag, nil, 0600)) + w := &worker{cfg: config.TraceArchiveConfig{Dir: dir, MaxLocalSizeMB: 1, Upload: config.TraceUploadConfig{Enabled: true, Binary: fake, Remote: remote}}} + s, err := newSegment(dir, "2000-01-01") + check(t, err) + check(t, s.seal(w.cfg)) + live, err := newSegment(dir, today()) + check(t, err) + check(t, live.f.Truncate(2<<20)) + check(t, live.f.Close()) + if err = w.maintenance(context.Background()); err == nil { + t.Fatal("accepted corrupt remote") + } + ms, err := listManifests(dir) + check(t, err) + if len(ms) != 1 || ms[0].UploadedTo != "" { + t.Fatal("deleted unconfirmed archive") + } + check(t, os.Remove(flag)) + check(t, w.maintenance(context.Background())) + ms, err = listManifests(dir) + check(t, err) + if len(ms) != 0 { + t.Fatal("retained confirmed archive") + } + _, err = os.Stat(live.path) + check(t, err) + ms, err = listManifests(remote) + check(t, err) + if len(ms) != 1 || ms[0].UploadedTo != remote { + t.Fatalf("unconfirmed remote manifest: %+v", ms) + } + check(t, readArchive(remote, "", "", "verify", "calls", nil, io.Discard)) +} diff --git a/internal/tracearchive/capture.go b/internal/tracearchive/capture.go new file mode 100644 index 00000000000..d20a2766f0f --- /dev/null +++ b/internal/tracearchive/capture.go @@ -0,0 +1,383 @@ +package tracearchive + +import ( + "bytes" + "context" + "io" + "net/http" + "slices" + "strings" + "sync" + "time" + + "github.com/MoeForge/moego/base64" + "github.com/MoeForge/moego/ojson" + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" + "github.com/rune-oai/t3code-proxy/internal/buildinfo" + "github.com/rune-oai/t3code-proxy/sdk/cliproxy/usage" +) + +// Only these selected fields cross the archive boundary. In particular neither +// usage.Record (which contains the client API key) nor headers are serialized. +type Metadata struct { + SchemaVersion int `json:"schema_version"` + HubVersion string `json:"hub_version"` + Start time.Time `json:"start"` + FirstByte *time.Time `json:"first_byte,omitempty"` + End time.Time `json:"end"` + LatencyNS int64 `json:"latency_ns"` + TTFTNS int64 `json:"ttft_ns"` + Endpoint string `json:"endpoint"` + Protocol string `json:"client_protocol"` + UserAgent string `json:"user_agent"` + ClientName string `json:"client_name,omitempty"` + Transport string `json:"transport"` + Stream bool `json:"stream"` + Session string `json:"session_id,omitempty"` + ParentSession string `json:"parent_session_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + RequestID string `json:"request_id"` + Provider string `json:"provider,omitempty"` + Executor string `json:"executor,omitempty"` + Account string `json:"auth_index,omitempty"` + Model string `json:"requested_model,omitempty"` + Alias string `json:"alias,omitempty"` + UpstreamModel string `json:"upstream_model,omitempty"` + ResponseModel string `json:"response_model,omitempty"` + Effort string `json:"reasoning_effort,omitempty"` + Tier string `json:"service_tier,omitempty"` + Status int `json:"status"` + Failed bool `json:"failed"` + FailureCode string `json:"failure_code,omitempty"` + Attempts []Attempt `json:"attempts"` + Tokens *usage.Detail `json:"token_usage,omitempty"` +} +type Attempt struct { + Start time.Time `json:"start"` + Account string `json:"auth_index,omitempty"` + Failed bool `json:"failed,omitempty"` + Code string `json:"failure_code,omitempty"` + Provider string `json:"provider"` + Transport string `json:"transport"` + Status int `json:"status,omitempty"` +} + +type call struct { + reserved int64 + Metadata + client, upstream, downstream []byte + chunks [][]byte + encoding string +} +type captureKey struct{} +type sourceKey struct{} +type source struct { + rec *Recorder + meta Metadata + encoding string +} + +func (r *Recorder) Bind(req *http.Request, protocol, traceID string) context.Context { + m := Metadata{SchemaVersion: 1, HubVersion: buildinfo.Version, Start: time.Now().UTC(), Endpoint: req.URL.Path, Protocol: protocol, UserAgent: req.UserAgent(), ClientName: req.Header.Get("X-Client-Name"), TraceID: traceID, Transport: "http"} + return context.WithValue(req.Context(), sourceKey{}, source{r, m, req.Header.Get("Content-Encoding")}) +} + +type Capture struct { + mu sync.Mutex // protects only this call's snapshots, never storage or queue waits + run *worker + source source + finished bool + data *call +} + +const maxCaptureBytes = 128 << 20 + +func Begin(ctx context.Context, ws bool, body []byte) (context.Context, *Capture) { + src, ok := ctx.Value(sourceKey{}).(source) + if !ok { + return ctx, nil + } + w := src.rec.active.Load() + if w == nil || slices.Contains(w.cfg.ExcludeEndpoints, src.meta.Endpoint) { + return ctx, nil + } + select { + case w.slots <- struct{}{}: + default: + w.dropped.Add(1) + return ctx, nil + } + m := src.meta + m.RequestID = uuid.NewString() + if ws { + m.Transport = "ws" + m.Stream = true + m.Start = time.Now().UTC() + } + c := &Capture{run: w, source: src, data: &call{Metadata: m, encoding: src.encoding}} + c.Client(body) + return c.attach(ctx), c +} + +// Context routes duplex frame and usage observations to the owning turn. +func (c *Capture) Context(ctx context.Context) context.Context { + if From(ctx) == c { + return ctx + } + if c == nil { + return usage.WithRecordObserver(context.WithValue(ctx, captureKey{}, c), nil) + } + return c.attach(ctx) +} +func (c *Capture) attach(ctx context.Context) context.Context { + ctx = context.WithValue(ctx, sourceKey{}, c.source) + return usage.WithRecordObserver(context.WithValue(ctx, captureKey{}, c), c.observe) +} +func From(ctx context.Context) *Capture { + if ctx == nil { + return nil + } + c, _ := ctx.Value(captureKey{}).(*Capture) + return c +} +func Inherit(ctx, src context.Context) context.Context { + if From(ctx) == nil { + if c := From(src); c != nil { + return c.attach(ctx) + } + if s, ok := src.Value(sourceKey{}).(source); ok { + return context.WithValue(ctx, sourceKey{}, s) + } + } + return ctx +} +func (c *Capture) reserve(n int) bool { + if c.data == nil { + return false + } + c.data.reserved += int64(n) + total := c.run.memory.Add(int64(n)) + if c.data.reserved > maxCaptureBytes || total > 256<<20 { + c.run.memory.Add(-c.data.reserved) + c.data = nil + c.run.dropped.Add(1) + return false + } + + return true +} +func (c *Capture) Client(p []byte) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.reserve(len(p)) { + c.data.client = append(c.data.client, p...) + } +} +func (c *Capture) Request(provider, transport string, p []byte) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.reserve(len(p)) { + d := c.data + d.Provider = provider + d.Attempts = append(d.Attempts, Attempt{Start: time.Now().UTC(), Provider: provider, Transport: transport}) + d.upstream = bytes.Clone(p) + d.chunks = nil + } +} +func (c *Capture) Status(status int) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if d := c.data; d != nil && len(d.Attempts) > 0 { + d.Attempts[len(d.Attempts)-1].Status = status + } +} +func (c *Capture) FirstByte() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.data != nil { + c.data.markFirstByte() + } +} +func (d *call) markFirstByte() { + if d.FirstByte != nil { + return + } + now := time.Now().UTC() + d.FirstByte = &now + d.TTFTNS = now.Sub(d.Start).Nanoseconds() +} +func (c *Capture) chunk(p []byte, upstream bool) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + if c.reserve(len(p) + 24) { + d := c.data + if len(p) > 0 { + d.markFirstByte() + } + if upstream { + d.chunks = append(d.chunks, bytes.Clone(p)) + } else { + d.downstream = append(d.downstream, p...) + } + } +} +func (c *Capture) Chunk(p []byte) { c.chunk(p, true) } +func (c *Capture) ClientResponse(p []byte) { c.chunk(p, false) } +func (c *Capture) observe(r usage.Record) { + c.mu.Lock() + defer c.mu.Unlock() + d := c.data + if d == nil { + return + } + d.RequestID = r.RequestID + if r.TraceID != "" { + d.TraceID = r.TraceID + } + d.Session = r.SessionID + d.ParentSession = r.ParentSessionID + d.Executor = r.ExecutorType + d.Account = r.AuthIndex + if n := len(d.Attempts); n > 0 { + a := &d.Attempts[n-1] + a.Account = r.AuthIndex + a.Failed = r.Failed + a.Code = r.Fail.Code + if a.Status == 0 && r.Fail.StatusCode > 0 { + a.Status = r.Fail.StatusCode + } + } + d.Alias = r.Alias + d.ResponseModel = r.ResponseModel + d.Effort = r.ReasoningEffort + d.Tier = r.ServiceTier + d.Failed = r.Failed + d.FailureCode = r.Fail.Code + d.Tokens = &r.Detail + if r.TTFT > 0 { + d.TTFTNS = r.TTFT.Nanoseconds() + } + if r.Provider != "" { + d.Provider = r.Provider + } + d.Model = r.Model +} +func (c *Capture) Finish(status int) { + if c == nil { + return + } + c.mu.Lock() + if c.finished { + c.mu.Unlock() + return + } + c.finished = true + d := c.data + c.data = nil + c.mu.Unlock() + if d == nil { + <-c.run.slots + return + } + d.End = time.Now().UTC() + d.LatencyNS = d.End.Sub(d.Start).Nanoseconds() + d.Status = status + d.Failed = d.Failed || status >= 400 + c.run.submit(d) +} + +func (d *call) prepare() { + if strings.EqualFold(d.encoding, "zstd") && !ojson.ValidBytes(d.client) { + if dec, err := zstd.NewReader(bytes.NewReader(d.client), zstd.WithDecoderMaxMemory(maxCaptureBytes)); err == nil { + if b, e := io.ReadAll(io.LimitReader(dec, maxCaptureBytes+1)); e == nil && len(b) <= maxCaptureBytes { + d.client = b + } + dec.Close() + } + } + if m := ojson.GetBytesNoCopy(d.client, "model").String(); m != "" { + d.Model = m + } + d.Stream = d.Stream || ojson.GetBytesNoCopy(d.client, "stream").Bool() + d.UpstreamModel = ojson.GetBytesNoCopy(d.upstream, "model").String() +} + +func (d *call) encode(put func([]byte) (string, error)) ([]byte, error) { + doc := entry("call") + _ = doc.Set("call", d.Metadata) + for _, v := range []struct { + name string + b []byte + }{{"client_request", d.client}, {"upstream_request", d.upstream}} { + if len(v.b) == 0 { + continue + } + packed, err := pack(v.b, put) + if err != nil { + return nil, err + } + _ = doc.SetRawBytes("call."+v.name, packed.Bytes()) + } + chunks := d.chunks + if len(chunks) == 0 && len(d.downstream) > 0 { + chunks = [][]byte{d.downstream} + } + if response, ok := reconstruct(chunks); ok { + _ = doc.SetRawBytes("call.response", canonical(nil, ojson.ParseBytes(response))) + if d.ResponseModel == "" { + _ = doc.Set("call.response_model", ojson.GetBytesNoCopy(response, "model").String()) + } + } else if len(chunks) > 0 { + frames := make([]string, len(chunks)) + for i, p := range chunks { + frames[i] = base64.StdEncoding.EncodeToString(redactFrame(p)) + } + _ = doc.Set("call.raw_frames_base64", frames) + } + // Some translators change content or model IDs. Preserve the downstream + // result as well when it differs from the provider-native replay material. + if len(d.chunks) > 0 && len(d.downstream) > 0 { + if response, ok := reconstruct([][]byte{d.downstream}); ok { + value := canonical(nil, ojson.ParseBytes(response)) + if !bytes.Equal(value, []byte(doc.Get("call.response").Raw)) { + _ = doc.SetRawBytes("call.client_response", value) + } + } else { + _ = doc.Set("call.client_raw_base64", base64.StdEncoding.EncodeToString(redactFrame(d.downstream))) + } + } + return doc.Bytes(), nil +} + +func redactFrame(p []byte) []byte { + if ojson.ValidBytes(p) { + return safePayload(p) + } + var b bytes.Buffer + for line := range bytes.SplitAfterSeq(p, []byte("\n")) { + if raw, ok := bytes.CutPrefix(line, []byte("data:")); ok && ojson.ValidBytes(raw) { + b.WriteString("data:") + b.Write(safePayload(raw)) + } else { + b.Write(line) + } + } + return b.Bytes() +} diff --git a/internal/tracearchive/cli.go b/internal/tracearchive/cli.go new file mode 100644 index 00000000000..535f1d534e6 --- /dev/null +++ b/internal/tracearchive/cli.go @@ -0,0 +1,215 @@ +package tracearchive + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "filippo.io/age" + "github.com/MoeForge/moego/ojson" + "github.com/klauspost/compress/zstd" + "github.com/rune-oai/t3code-proxy/internal/config" +) + +func Main(args []string, out, stderr io.Writer) int { + if err := command(args, out, stderr); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + fmt.Fprintln(stderr, "traces:", err) + return 1 + } + return 0 +} +func command(args []string, out, stderr io.Writer) error { + if len(args) == 0 || (args[0] != "export" && args[0] != "verify" && args[0] != "stats") { + return fmt.Errorf("usage: traces {export|verify|stats} [--dir PATH] [--from YYYY-MM-DD] [--to YYYY-MM-DD] [--identity FILE]") + } + mode := args[0] + fs := flag.NewFlagSet("traces "+mode, flag.ContinueOnError) + fs.SetOutput(stderr) + dir := fs.String("dir", "", "archive directory") + cfgPath := fs.String("config", "config.yaml", "hub config to resolve archive directory") + from := fs.String("from", "", "first UTC segment date (inclusive)") + to := fs.String("to", "", "last UTC segment date (inclusive)") + identity := fs.String("identity", "", "age identity file") + format := fs.String("format", "calls", "calls or conversations") + output := fs.String("output", "", "write to a new file (default stdout)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + if fs.NArg() != 0 || (*from != "" && !validDate(*from)) || (*to != "" && !validDate(*to)) || (*to != "" && *from > *to) || (*format != "calls" && *format != "conversations") { + return errors.New("invalid arguments or UTC date range") + } + if *dir == "" { + cfg := &config.Config{} + b, err := os.ReadFile(*cfgPath) + if err == nil { + cfg, err = config.ParseConfigBytes(b) + } else if errors.Is(err, os.ErrNotExist) && *cfgPath == "config.yaml" { + err = nil + } + if err != nil { + return err + } + *dir, err = Directory(cfg) + if err != nil { + return err + } + } + var ids []age.Identity + if *identity != "" { + b, err := os.ReadFile(*identity) + if err != nil { + return err + } + ids, err = age.ParseIdentities(bytes.NewReader(b)) + if err != nil { + return errors.New("invalid age identity file") + } + } + if *output != "" { + f, err := os.OpenFile(*output, os.O_WRONLY|os.O_EXCL|os.O_CREATE, 0600) + if err != nil { + return err + } + err = readArchive(*dir, *from, *to, mode, *format, ids, f) + if e := f.Close(); err == nil { + err = e + } + if err != nil { + if e := os.Remove(*output); e != nil { + return errors.Join(err, e) + } + } + return err + } + return readArchive(*dir, *from, *to, mode, *format, ids, out) +} + +func readArchive(dir, from, to, mode, format string, ids []age.Identity, out io.Writer) error { + manifests, err := listManifests(dir) + if err != nil { + return err + } + ms := map[string]Manifest{} + for _, m := range manifests { + ms[m.File] = m + } + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + groups := make(map[string][][]byte) + var visit visitor + if mode == "export" { + visit = func(b []byte) error { + if format == "calls" { + _, err := out.Write(append(b, '\n')) + return err + } + key := ojson.GetBytesNoCopy(b, "session_id").String() + if key == "" { + key = ojson.GetBytesNoCopy(b, "request_id").String() + } + groups[key] = append(groups[key], b) + return nil + } + } + for _, e := range entries { + m := segmentName.FindStringSubmatch(e.Name()) + if m == nil || (from != "" && m[1] < from) || (to != "" && m[1] > to) { + continue + } + manifest, ok := ms[e.Name()] + sealed := m[3] != "live" + if sealed && !ok { + return fmt.Errorf("%s: missing manifest", e.Name()) + } + delete(ms, e.Name()) + c := manifest.Counts + if mode != "stats" || !sealed { + l, err := readSegment(filepath.Join(dir, e.Name()), m[1], sealed, manifest, ids, visit) + if err != nil { + return fmt.Errorf("%s: %w", e.Name(), err) + } + c = l.counts + } + switch mode { + case "verify": + if _, err = fmt.Fprintf(out, "ok %s (%d calls, %d blobs)\n", e.Name(), c.Calls, c.Blobs); err != nil { + return err + } + case "stats": + d := object() + _ = d.Set("file", e.Name()) + _ = d.Set("counts", c) + _ = d.Set("dedup_ratio", c.ratio()) + _, err = out.Write(append(d.Bytes(), '\n')) + if err != nil { + return err + } + } + } + for _, m := range ms { + if (from == "" || m.Date >= from) && (to == "" || m.Date <= to) { + return fmt.Errorf("%s: manifest has no segment", m.File) + } + } + for _, key := range sortedKeys(groups) { + d := object() + _ = d.Set("session_id", ojson.GetBytesNoCopy(groups[key][0], "session_id").String()) + _ = d.SetRawBytes("calls", append(append([]byte{'['}, bytes.Join(groups[key], []byte(","))...), ']')) + if _, err = out.Write(append(d.Bytes(), '\n')); err != nil { + return err + } + } + return nil +} + +func readSegment(path, date string, sealed bool, m Manifest, ids []age.Identity, visit visitor) (*ledger, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer closeLogged(f) + info, err := f.Stat() + if err != nil { + return nil, err + } + var r io.Reader = io.LimitReader(f, info.Size()) + if sealed { + h, n, err := fileHash(path) + if err != nil { + return nil, err + } + if h != m.SHA256 || n != m.Size { + return nil, errors.New("sealed file does not match manifest") + } + if strings.HasSuffix(path, ".age") { + if len(ids) == 0 { + return nil, errors.New("--identity required") + } + r, err = age.Decrypt(r, ids...) + if err != nil { + return nil, err + } + } + z, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(1)) + if err != nil { + return nil, err + } + defer z.Close() + r = z + } + l, err := scan(r, date, sealed, visit) + if err == nil && sealed && (m.SourceSHA256 != l.checksum() || m.Counts != l.counts) { + return nil, errors.New("manifest counts or source hash mismatch") + } + return l, err +} diff --git a/internal/tracearchive/json.go b/internal/tracearchive/json.go new file mode 100644 index 00000000000..5af2518eadb --- /dev/null +++ b/internal/tracearchive/json.go @@ -0,0 +1,207 @@ +// Package tracearchive stores self-contained, versioned model-call segments. +package tracearchive + +import ( + "bytes" + "crypto/sha256" + "fmt" + "sort" + "strings" + + "github.com/MoeForge/moego/base64" + "github.com/MoeForge/moego/ojson" +) + +func digest(data []byte) string { return fmt.Sprintf("%x", sha256.Sum256(data)) } + +// Bound recursive JSON walks independently of call size. This scan also works +// on individual SSE data lines without interpreting quoted prompt text. +func withinDepth(data []byte) bool { + depth := 0 + quoted, escaped := false, false + for _, c := range data { + if quoted { + switch { + case escaped: + escaped = false + case c == '\\': + escaped = true + case c == '"': + quoted = false + } + continue + } + switch c { + case '"': + quoted = true + case '{', '[': + depth++ + if depth > 256 { + return false + } + case '}', ']': + if depth > 0 { + depth-- + } + } + } + return true +} + +// Canonical form sorts object names, normalizes string escapes and retains number +// lexemes (especially integers outside the exact range of a float64). +func canonical(dst []byte, r ojson.Result) []byte { + switch { + case r.IsObject(): + keys := r.Map() + names := make([]string, 0, len(keys)) + for key := range keys { + names = append(names, key) + } + sort.Strings(names) + dst = append(dst, '{') + for i, key := range names { + if i != 0 { + dst = append(dst, ',') + } + dst = ojson.AppendJSONString(dst, key) + dst = append(dst, ':') + dst = canonical(dst, keys[key]) + } + return append(dst, '}') + case r.IsArray(): + dst = append(dst, '[') + for i, item := range r.Array() { + if i != 0 { + dst = append(dst, ',') + } + dst = canonical(dst, item) + } + return append(dst, ']') + case r.Type == ojson.String: + return ojson.AppendJSONString(dst, r.String()) + default: + return append(dst, r.Raw...) + } +} + +func object() *ojson.Doc { return ojson.NewDocString("{}") } +func entry(kind string) *ojson.Doc { d := object(); _ = d.Set("type", kind); return d } + +func credentialHeader(name string) bool { + name = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(name), "_", "-")) + return strings.Contains(name, "authorization") || strings.Contains(name, "api-key") || strings.Contains(name, "apikey") || strings.Contains(name, "cookie") || name == "token" || strings.HasSuffix(name, "-token") +} +func safePayload(data []byte) []byte { + if !ojson.ValidBytes(data) { + return data + } + d := ojson.NewDoc(data) + // WS frames can carry transport metadata inside the JSON envelope. Treat + // these containers as headers even if a provider uses unusual key casing. + var walk func(ojson.Result, string, bool) + walk = func(r ojson.Result, path string, headers bool) { + if !r.IsObject() && !r.IsArray() { + return + } + named := headers && (credentialHeader(r.Get("name").String()) || credentialHeader(r.Get("key").String())) + tuple := headers && r.IsArray() && credentialHeader(r.Get("0").String()) + index := 0 + r.ForEach(func(k, v ojson.Result) bool { + key := k.String() + if r.IsArray() { + key = fmt.Sprint(index) + index++ + } + p := ojson.Escape(key) + if path != "" { + p = path + "." + p + } + name := strings.ToLower(strings.ReplaceAll(key, "_", "-")) + lineHeader, _, hasColon := strings.Cut(v.Str, ":") + if headers && (credentialHeader(name) || named && (name == "value" || name == "values") || tuple && key != "0" || hasColon && credentialHeader(lineHeader)) { + _ = d.Set(p, "[masked]") + } else { + walk(v, p, headers || strings.Contains(name, "headers") || name == "client-metadata" || name == "metadata") + } + return true + }) + } + walk(ojson.ParseBytes(data), "", false) + return d.Bytes() +} + +// A separate parts object makes references unambiguous even when user data +// contains an object that looks exactly like an archive reference. +func pack(data []byte, put func([]byte) (string, error)) (*ojson.Doc, error) { + d := object() + if !ojson.ValidBytes(data) { + _ = d.Set("base64", base64.StdEncoding.EncodeToString(data)) + return d, nil + } + body := ojson.NewDoc(safePayload(data)) + for _, key := range []string{"system", "instructions", "tools", "messages", "input", "contents", "systemInstruction", "toolConfig"} { + v := body.Get(key) + if !v.Exists() { + continue + } + if v.IsArray() { + hashes := make([]string, 0) + for _, item := range v.Array() { + h, err := put(canonical(nil, item)) + if err != nil { + return nil, err + } + hashes = append(hashes, h) + } + _ = d.Set("parts."+key, hashes) + } else { + h, err := put(canonical(nil, v)) + if err != nil { + return nil, err + } + _ = d.Set("parts."+key, h) + } + _ = body.Delete(key) + } + _ = d.SetRawBytes("body", canonical(nil, ojson.ParseBytes(body.Bytes()))) + return d, nil +} + +func unpack(p ojson.Result, blobs map[string][]byte) ([]byte, error) { + if p.Get("base64").Exists() { + return []byte(p.Raw), nil + } + body := p.Get("body") + if !body.Exists() { + return nil, fmt.Errorf("missing packed body") + } + d := ojson.NewDocString(body.Raw) + for key, v := range p.Get("parts").Map() { + var b bytes.Buffer + if v.IsArray() { + b.WriteByte('[') + } + refs := []ojson.Result{v} + if v.IsArray() { + refs = v.Array() + } + for i, r := range refs { + data, ok := blobs[r.String()] + if !ok { + return nil, fmt.Errorf("missing blob %s", r.String()) + } + if i > 0 { + b.WriteByte(',') + } + b.Write(data) + } + if v.IsArray() { + b.WriteByte(']') + } + if err := d.SetRawBytes(ojson.Escape(key), b.Bytes()); err != nil { + return nil, err + } + } + return d.Bytes(), nil +} diff --git a/internal/tracearchive/recorder.go b/internal/tracearchive/recorder.go new file mode 100644 index 00000000000..7236d41dacf --- /dev/null +++ b/internal/tracearchive/recorder.go @@ -0,0 +1,413 @@ +package tracearchive + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/util" + log "github.com/sirupsen/logrus" + bolt "go.etcd.io/bbolt" +) + +type Recorder struct { + mu sync.Mutex + active atomic.Pointer[worker] + cfg config.TraceArchiveConfig + dropped atomic.Uint64 + lastError string +} +type worker struct { + memory atomic.Int64 + cfg config.TraceArchiveConfig + dropped *atomic.Uint64 + slots chan struct{} + queue chan *call + gate sync.RWMutex + closed bool + done chan struct{} + seal chan chan error + wake chan struct{} + maintenanceDone chan struct{} + cancel context.CancelFunc + lock *bolt.DB + segment *segment // owned by writer goroutine + statusMu sync.Mutex + live counts + date, lastError, lastUpload string + failed atomic.Bool +} + +func Directory(c *config.Config) (string, error) { + if c.TraceArchive.Dir != "" { + return filepath.Abs(c.TraceArchive.Dir) + } + dir, err := util.ResolveAuthDir(c.AuthDir) + if err != nil { + return "", err + } + return filepath.Join(dir, "traces"), nil +} +func (r *Recorder) Apply(c *config.Config) error { + r.mu.Lock() + defer r.mu.Unlock() + cfg := c.TraceArchive + dir, err := Directory(c) + if err != nil { + return err + } + cfg.Dir = dir + if cfg.QueueSize == 0 { + cfg.QueueSize = 32 + } + if w := r.active.Load(); w != nil && reflect.DeepEqual(cfg, w.cfg) && !w.failed.Load() { + return nil + } + r.stop() + r.cfg = cfg + r.lastError = "" + if !cfg.Enabled { + return nil + } + if err = cfg.Validate(); err == nil { + err = r.start(cfg) + } + if err != nil { + r.lastError = err.Error() + } + return err +} +func (r *Recorder) Enabled() bool { w := r.active.Load(); return w != nil && !w.failed.Load() } +func (r *Recorder) start(cfg config.TraceArchiveConfig) error { + if err := os.MkdirAll(cfg.Dir, 0700); err != nil { + return err + } + lock, err := bolt.Open(filepath.Join(cfg.Dir, ".writer.lock"), 0600, &bolt.Options{Timeout: time.Millisecond}) + if err != nil { + return fmt.Errorf("open trace archive writer lock: %w", err) + } + w := &worker{cfg: cfg, lock: lock, dropped: &r.dropped, slots: make(chan struct{}, cfg.QueueSize), queue: make(chan *call, cfg.QueueSize), seal: make(chan chan error), done: make(chan struct{}), wake: make(chan struct{}, 1), maintenanceDone: make(chan struct{})} + if err = w.recover(); err != nil { + if w.segment != nil && w.segment.f != nil { + closeLogged(w.segment.f) + } + closeLogged(lock) + return err + } + ctx, cancel := context.WithCancel(context.Background()) + w.cancel = cancel + w.snapshot() + go w.run() + go w.maintain(ctx) + r.active.Store(w) + return nil +} +func (w *worker) recover() error { + entries, err := os.ReadDir(w.cfg.Dir) + if err != nil { + return err + } + ms, err := listManifests(w.cfg.Dir) + if err != nil { + return err + } + for _, e := range entries { + // Only this process holds the directory writer lock. + if strings.HasPrefix(e.Name(), ".trace-tmp-") && e.Type().IsRegular() { + if err := os.Remove(filepath.Join(w.cfg.Dir, e.Name())); err != nil { + return err + } + continue + } + m := segmentName.FindStringSubmatch(e.Name()) + if m == nil || m[3] != "live" { + continue + } + if w.segment != nil { + return errors.New("multiple current live segments") + } + s, err := recoverSegment(filepath.Join(w.cfg.Dir, e.Name()), m[1]) + if err != nil { + return err + } + w.segment = s + target := strings.TrimSuffix(s.path, ".live") + ".jsonl.zst" + _, plainErr := os.Stat(target) + _, ageErr := os.Stat(target + ".age") + committed := false + for _, manifest := range ms { + dest := filepath.Join(w.cfg.Dir, manifest.File) + if dest != target && dest != target+".age" { + continue + } + h, n, err := fileHash(dest) + if err != nil { + return err + } + if h != manifest.SHA256 || n != manifest.Size || manifest.SourceSHA256 != s.checksum() { + return errors.New("live and committed trace segment disagree") + } + if err = s.f.Close(); err != nil { + return err + } + s.f = nil + if err = os.Remove(s.path); err != nil { + return err + } + w.segment = nil + committed = true + break + } + if committed { + continue + } + if m[1] != today() || plainErr == nil || ageErr == nil { + if err = w.sealSegment(); err != nil { + return err + } + // A crash before manifest publication can leave an orphan with the old + // encryption suffix. The replacement is durable before removing the orphan. + orphan := target + ".age" + if len(w.cfg.Recipients) > 0 { + orphan = target + } + if err = os.Remove(orphan); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + } + return nil +} +func today() string { return time.Now().UTC().Format(time.DateOnly) } +func (r *Recorder) Close() { r.mu.Lock(); defer r.mu.Unlock(); r.stop() } +func (r *Recorder) stop() { + w := r.active.Swap(nil) + if w == nil { + return + } + w.gate.Lock() + w.closed = true + close(w.queue) + w.gate.Unlock() + w.cancel() + <-w.done + <-w.maintenanceDone + closeLogged(w.lock) +} +func (w *worker) submit(d *call) { + if w.gate.TryRLock() { + if !w.closed && !w.failed.Load() { + select { + case w.queue <- d: + w.gate.RUnlock() + return + default: + } + } + w.gate.RUnlock() + } + w.memory.Add(-d.reserved) + <-w.slots + w.dropped.Add(1) +} +func (w *worker) snapshot() { + w.statusMu.Lock() + defer w.statusMu.Unlock() + w.live = counts{} + w.date = "" + if w.segment != nil { + w.live = w.segment.counts + w.date = w.segment.date + } +} +func (w *worker) sealSegment() error { + if w.segment == nil { + return nil + } + if err := w.segment.seal(w.cfg); err != nil { + return err + } + w.segment = nil + w.snapshot() + select { + case w.wake <- struct{}{}: + default: + } + return nil +} +func (w *worker) roll() error { + if w.segment != nil && w.segment.date != today() { + return w.sealSegment() + } + return nil +} +func (w *worker) record(d *call) error { + d.prepare() + if !withinDepth(d.client) || !withinDepth(d.upstream) || !withinDepth(d.downstream) || slices.ContainsFunc(d.chunks, func(p []byte) bool { return !withinDepth(p) }) { + w.dropped.Add(1) + return nil + } + if slices.Contains(w.cfg.ExcludeModels, d.Model) { + return nil + } + if err := w.roll(); err != nil { + return err + } + if w.segment == nil { + s, err := newSegment(w.cfg.Dir, today()) + if err != nil { + return err + } + w.segment = s + } + err := w.segment.append(d) + w.snapshot() + return err +} +func (w *worker) fail(err error) { + if err != nil { + w.failed.Store(true) + w.statusMu.Lock() + w.lastError = err.Error() + w.statusMu.Unlock() + log.WithError(err).Error("trace archive writer stopped") + } +} +func (w *worker) consume(d *call) { + if w.failed.Load() { + w.dropped.Add(1) + } else if err := w.record(d); err != nil { + w.dropped.Add(1) + w.fail(err) + } + w.memory.Add(-d.reserved) + <-w.slots +} +func (w *worker) run() { + defer close(w.done) + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + defer func() { + if w.segment != nil && w.segment.f != nil { + closeLogged(w.segment.f) + } + }() + for { + select { + case d, ok := <-w.queue: + if !ok { + if !w.failed.Load() { + w.fail(w.roll()) + } + return + } + w.consume(d) + case answer := <-w.seal: + if w.failed.Load() { + answer <- errors.New("trace writer failed") + continue + } + // Flush completed calls already admitted before this management operation. + for n := len(w.queue); n > 0; n-- { + d := <-w.queue + if d == nil { + break + } + w.consume(d) + if w.failed.Load() { + break + } + } + var err error + if w.failed.Load() { + err = errors.New("trace writer failed") + } else { + err = w.sealSegment() + w.fail(err) + } + answer <- err + case <-ticker.C: + if !w.failed.Load() { + w.fail(w.roll()) + } + } + } +} +func (r *Recorder) Seal(ctx context.Context) error { + w := r.active.Load() + if w == nil { + return errors.New("trace archive disabled") + } + reply := make(chan error, 1) + select { + case w.seal <- reply: + case <-ctx.Done(): + return ctx.Err() + case <-w.done: + return errors.New("trace archive stopped") + } + select { + case err := <-reply: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +type Status struct { + Enabled bool `json:"enabled"` + Queue int `json:"queue_depth"` + Dropped uint64 `json:"dropped"` + Calls int64 `json:"today_calls"` + Bytes int64 `json:"today_bytes"` + Ratio float64 `json:"dedup_ratio"` + Segments []Manifest `json:"segments"` + Error string `json:"last_error,omitempty"` + UploadError string `json:"last_upload_error,omitempty"` +} + +func (r *Recorder) Status() Status { + r.mu.Lock() + defer r.mu.Unlock() + s := Status{Dropped: r.dropped.Load(), Error: r.lastError, Segments: []Manifest{}, Ratio: 1} + var c counts + if w := r.active.Load(); w != nil { + s.Enabled = !w.failed.Load() + s.Queue = len(w.queue) + w.statusMu.Lock() + if w.date == today() { + c = w.live + } + s.Error = w.lastError + s.UploadError = w.lastUpload + w.statusMu.Unlock() + } + if dir := r.cfg.Dir; dir != "" { + ms, err := listManifests(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + s.Error = err.Error() + } + s.Segments = append(s.Segments, ms...) + for _, m := range ms { + if m.Date == today() { + c.Calls += m.Counts.Calls + c.Bytes += m.Counts.Bytes + c.Unique += m.Counts.Unique + c.Referenced += m.Counts.Referenced + } + } + } + s.Calls = c.Calls + s.Bytes = c.Bytes + s.Ratio = c.ratio() + return s +} diff --git a/internal/tracearchive/response.go b/internal/tracearchive/response.go new file mode 100644 index 00000000000..ee9a691dda9 --- /dev/null +++ b/internal/tracearchive/response.go @@ -0,0 +1,310 @@ +package tracearchive + +import ( + "bytes" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/MoeForge/moego/ojson" +) + +// Executor hooks supply either a response body, scanner lines or WS messages. +// A terminal provider event is required before discarding the original frames. +func reconstruct(chunks [][]byte) ([]byte, bool) { + if len(chunks) == 0 { + return nil, false + } + if len(chunks) == 1 && ojson.ValidBytes(chunks[0]) { + r := ojson.ParseBytes(chunks[0]) + typ := r.Get("type").String() + if !strings.HasPrefix(typ, "response.") && !strings.HasPrefix(typ, "content_block_") && !strings.HasPrefix(typ, "message_") && !r.Get("choices.0.delta").Exists() && typ != "ping" { + return safePayload(chunks[0]), true + } + } + var s streamState + for _, chunk := range chunks { + if ojson.ValidBytes(chunk) { + if !s.event(ojson.ParseBytes(safePayload(chunk))) { + return nil, false + } + continue + } + for line := range bytes.SplitSeq(chunk, []byte("\n")) { + line = bytes.TrimSpace(line) + if len(line) == 0 || bytes.HasPrefix(line, []byte(":")) || bytes.HasPrefix(line, []byte("event:")) || bytes.HasPrefix(line, []byte("id:")) { + continue + } + raw, ok := bytes.CutPrefix(line, []byte("data:")) + if !ok { + return nil, false + } + raw = bytes.TrimSpace(raw) + if bytes.Equal(raw, []byte("[DONE]")) { + s.chatEnd = true + continue + } + if !ojson.ValidBytes(raw) || !s.event(ojson.ParseBytes(safePayload(raw))) { + return nil, false + } + } + } + if s.doc == nil || !(s.complete || s.chat && s.chatEnd) || len(s.openBlocks) != 0 { + return nil, false + } + for path, b := range s.text { + if err := s.doc.Set(path, b.String()); err != nil { + return nil, false + } + } + return s.doc.Bytes(), true +} + +type streamState struct { + text map[string]*strings.Builder + doc *ojson.Doc + complete, chat, chatEnd bool + items map[int]string + observedItems map[int]bool + openBlocks map[int]*strings.Builder +} + +func (s *streamState) event(r ojson.Result) bool { + typ := r.Get("type").String() + idx := r.Get("output_index") + if strings.HasPrefix(typ, "response.") && (idx.Exists() || strings.HasSuffix(typ, ".delta")) { + n := idx.Int() + if n < 0 || n > 65535 { + return false + } + if s.observedItems == nil { + s.observedItems = make(map[int]bool) + } + s.observedItems[int(n)] = true + } + switch { + case typ == "ping": + return true + case typ == "message_start": + msg := r.Get("message") + if !msg.IsObject() || s.doc != nil { + return false + } + s.doc = ojson.NewDocString(msg.Raw) + s.openBlocks = make(map[int]*strings.Builder) + case typ == "content_block_start", typ == "content_block_delta", typ == "content_block_stop": + if s.doc == nil || s.openBlocks == nil { + return false + } + n := int(r.Get("index").Int()) + path := "content." + strconv.Itoa(n) + if typ == "content_block_start" { + b := r.Get("content_block") + if !b.IsObject() || n != len(s.doc.Get("content").Array()) { + return false + } + s.openBlocks[n] = new(strings.Builder) + return s.doc.SetRaw(path, b.Raw) == nil + } + partial := s.openBlocks[n] + if partial == nil { + return false + } + if typ == "content_block_stop" { + delete(s.openBlocks, n) + if partial.Len() > 0 { + if !ojson.Valid(partial.String()) { + return false + } + return s.doc.SetRaw(path+".input", partial.String()) == nil + } + return true + } + delta := r.Get("delta") + switch delta.Get("type").String() { + case "input_json_delta": + partial.WriteString(delta.Get("partial_json").String()) + case "citations_delta": + return s.doc.SetRaw(path+".citations.-1", delta.Get("citation").Raw) == nil + case "text_delta", "thinking_delta", "signature_delta": + key := strings.TrimSuffix(delta.Get("type").String(), "_delta") + return s.appendString(path+"."+key, delta.Get(key)) + default: + return false + } + case typ == "message_delta": + if s.doc == nil { + return false + } + mergeFields(s.doc, "", r.Get("delta")) + mergeFields(s.doc, "usage.", r.Get("usage")) + case typ == "message_stop": + s.complete = true + case typ == "response.output_item.done": + item := r.Get("item") + if !item.IsObject() { + return false + } + if s.items == nil { + s.items = make(map[int]string) + } + s.items[int(r.Get("output_index").Int())] = item.Raw + case typ == "response.completed", typ == "response.done", typ == "response.incomplete", typ == "response.failed": + resp := r.Get("response") + if !resp.IsObject() || s.complete { + return false + } + s.doc = ojson.NewDocString(resp.Raw) + for i, item := range s.items { + p := "output." + strconv.Itoa(i) + final := s.doc.Get(p) + if !final.IsObject() { + _ = s.doc.SetRaw(p, item) + continue + } + // Some compatible endpoints send a compact terminal item and + // report its opaque replay fields only on output_item.done. + prior := ojson.Parse(item) + if id := prior.Get("id").String(); id != "" && final.Get("id").Exists() && final.Get("id").String() != id { + return false + } + for k, v := range prior.Map() { + key := ojson.Escape(k) + if prev := final.Get(key); !prev.Exists() || prev.Type == ojson.Null || (prev.IsArray() && len(prev.Array()) == 0 && v.IsArray()) { + _ = s.doc.SetRaw(p+"."+key, v.Raw) + } + } + } + for i := range s.observedItems { + if !s.doc.Get("output." + strconv.Itoa(i)).IsObject() { + return false + } + } + s.complete = true + case strings.HasPrefix(typ, "response.") || typ == "codex.rate_limits": + // Responses terminal frames carry whole items, or reference indexed item.done frames. + case r.Get("choices").IsArray(): + return s.chatChunk(r) + default: + return false + } + return true +} + +func mergeFields(d *ojson.Doc, prefix string, r ojson.Result) { + for key, value := range r.Map() { + _ = d.SetRaw(prefix+ojson.Escape(key), value.Raw) + } +} +func (s *streamState) appendString(path string, v ojson.Result) bool { + if v.Type != ojson.String { + return false + } + if s.text == nil { + s.text = make(map[string]*strings.Builder) + } + b := s.text[path] + if b == nil { + b = new(strings.Builder) + b.WriteString(s.doc.Get(path).String()) + s.text[path] = b + } + b.WriteString(v.String()) + return true +} +func (s *streamState) chatChunk(r ojson.Result) bool { + if s.doc == nil { + s.doc = object() + s.chat = true + _ = s.doc.SetRaw("choices", "[]") + } + if !s.chat { + return false + } + for k, v := range r.Map() { + if k != "choices" && k != "object" { + if v.Type != ojson.Null || !s.doc.Get(ojson.Escape(k)).Exists() { + _ = s.doc.SetRaw(ojson.Escape(k), v.Raw) + } + } + } + _ = s.doc.Set("object", "chat.completion") + for _, choice := range r.Get("choices").Array() { + i := choice.Get("index").Int() + if i < 0 || i > 4096 { + return false + } + p := fmt.Sprintf("choices.%d.", i) + _ = s.doc.Set(p+"index", i) + if v := choice.Get("finish_reason"); v.Exists() && v.Type != ojson.Null { + _ = s.doc.SetRaw(p+"finish_reason", v.Raw) + } + if v := choice.Get("logprobs"); v.IsObject() { + for k, arr := range v.Map() { + for _, val := range arr.Array() { + _ = s.doc.SetRaw(p+"logprobs."+ojson.Escape(k)+".-1", val.Raw) + } + } + } + for k, v := range choice.Get("delta").Map() { + dest := p + "message." + ojson.Escape(k) + switch k { + case "role": + _ = s.doc.SetRaw(dest, v.Raw) + case "content", "refusal", "reasoning_content", "reasoning": + if v.Type == ojson.Null { + if !s.doc.Get(dest).Exists() { + _ = s.doc.SetRaw(dest, "null") + } + } else if !s.appendString(dest, v) { + return false + } + case "annotations": + for _, a := range v.Array() { + _ = s.doc.SetRaw(dest+".-1", a.Raw) + } + case "function_call": + for f, str := range v.Map() { + if !s.appendString(dest+"."+ojson.Escape(f), str) { + return false + } + } + case "tool_calls": + for _, tc := range v.Array() { + j := tc.Get("index").Int() + if j < 0 || j > 4096 { + return false + } + tp := fmt.Sprintf("%s.%d.", dest, j) + for f, x := range tc.Map() { + switch f { + case "index": + case "function": + for key, str := range x.Map() { + if !s.appendString(tp+"function."+ojson.Escape(key), str) { + return false + } + } + default: + _ = s.doc.SetRaw(tp+ojson.Escape(f), x.Raw) + } + } + } + default: + return false + } + } + } + return true +} + +// Stable ordering for filenames, session groups and other small string maps. +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/internal/tracearchive/segment.go b/internal/tracearchive/segment.go new file mode 100644 index 00000000000..b78d9a640ff --- /dev/null +++ b/internal/tracearchive/segment.go @@ -0,0 +1,423 @@ +package tracearchive + +import ( + "bufio" + "crypto/sha256" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "filippo.io/age" + "github.com/MoeForge/moego/ojson" + "github.com/klauspost/compress/zstd" + "github.com/rune-oai/t3code-proxy/internal/config" + log "github.com/sirupsen/logrus" +) + +var segmentName = regexp.MustCompile(`^traces-(\d{4}-\d{2}-\d{2})(-\d+)?\.(live|jsonl\.zst(\.age)?)$`) + +type counts struct { + Calls int64 `json:"calls"` + Blobs int64 `json:"blobs"` + Unique int64 `json:"unique_blob_bytes"` + Referenced int64 `json:"referenced_blob_bytes"` + Bytes int64 `json:"jsonl_bytes"` +} + +func (c counts) ratio() float64 { + if c.Unique == 0 { + return 1 + } + return float64(c.Referenced) / float64(c.Unique) +} + +type ledger struct { + counts + date string + sum hash.Hash + sizes map[string]int +} + +func newLedger() *ledger { return &ledger{sum: sha256.New(), sizes: make(map[string]int)} } +func (l *ledger) line(p []byte) { _, _ = l.sum.Write(p); l.Bytes += int64(len(p)) } +func (l *ledger) checksum() string { return fmt.Sprintf("%x", l.sum.Sum(nil)) } + +type segment struct { + *ledger + path string + f *os.File +} + +func closeLogged(c io.Closer) { + if err := c.Close(); err != nil { + log.WithError(err).Error("trace archive close failed") + } +} + +func newSegment(dir, date string) (*segment, error) { + path := filepath.Join(dir, fmt.Sprintf("traces-%s-%d.live", date, time.Now().UnixNano())) + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + s := &segment{newLedger(), path, f} + s.date = date + d := entry("header") + _ = d.Set("schema_version", 1) + _ = d.Set("date", date) + if err = s.write(d.Bytes()); err == nil { + err = syncDirectory(dir) + } + if err != nil { + closeLogged(f) + return nil, err + } + return s, nil +} +func (s *segment) write(data []byte) error { + line := append(data, '\n') + if _, err := s.f.Write(line); err != nil { + return err + } + s.line(line) + return nil +} +func (s *segment) append(d *call) error { + var referenced int64 + data, err := d.encode(func(b []byte) (string, error) { + h := digest(b) + referenced += int64(len(b)) + if _, ok := s.sizes[h]; ok { + return h, nil + } + e := entry("blob") + _ = e.Set("sha256", h) + _ = e.SetRawBytes("value", b) + if err := s.write(e.Bytes()); err != nil { + return "", err + } + s.sizes[h] = len(b) + s.Unique += int64(len(b)) + s.Blobs++ + return h, nil + }) + if err != nil { + return err + } + if err = s.write(data); err != nil { + return err + } + s.Calls++ + s.Referenced += referenced + return s.f.Sync() +} + +type visitor func([]byte) error + +// ReadBytes handles large lines. An unterminated last line is recoverable only +// in a live segment. Complete invalid lines always signal corruption. +func scan(r io.Reader, date string, sealed bool, visit visitor) (*ledger, error) { + l := newLedger() + blobs := map[string][]byte{} + reader := bufio.NewReader(r) + footer := false + for { + line, err := reader.ReadBytes('\n') + if err != nil { + if !errors.Is(err, io.EOF) { + return nil, err + } + if sealed && (!footer || len(line) > 0) { + return nil, fmt.Errorf("missing footer or torn sealed tail") + } + break + } + if footer || !ojson.ValidBytes(line) { + return nil, fmt.Errorf("invalid entry at byte %d", l.Bytes) + } + e := ojson.ParseBytes(line) + typ := e.Get("type").String() + if (l.Bytes == 0) != (typ == "header") { + return nil, fmt.Errorf("misplaced or missing header") + } + switch typ { + case "header": + if e.Get("schema_version").Int() != 1 || e.Get("date").String() != date { + return nil, fmt.Errorf("unsupported header or incorrect date") + } + l.date = date + case "blob": + v := e.Get("value") + if !v.Exists() { + return nil, fmt.Errorf("missing blob value") + } + b := canonical(nil, v) + h := e.Get("sha256").String() + if _, ok := l.sizes[h]; ok || digest(b) != h { + return nil, fmt.Errorf("duplicate or invalid blob at %d", l.Bytes) + } + l.sizes[h] = len(b) + if visit != nil { + blobs[h] = b + } + l.Blobs++ + l.Unique += int64(len(b)) + case "call": + c := e.Get("call") + if c.Get("schema_version").Int() != 1 { + return nil, fmt.Errorf("unsupported call schema") + } + var resolved *ojson.Doc + if visit != nil { + resolved = ojson.NewDocString(c.Raw) + } + for _, key := range []string{"client_request", "upstream_request"} { + p := c.Get(key) + if !p.Exists() { + continue + } + for _, part := range p.Get("parts").Map() { + for _, ref := range part.Array() { + n, ok := l.sizes[ref.String()] + if !ok { + return nil, fmt.Errorf("missing reference at byte %d", l.Bytes) + } + l.Referenced += int64(n) + } + } + if visit != nil { + v, err := unpack(p, blobs) + if err != nil { + return nil, err + } + _ = resolved.SetRawBytes(key, v) + } + } + if visit != nil { + if err := visit(resolved.Bytes()); err != nil { + return nil, err + } + } + l.Calls++ + case "footer": + if !sealed || e.Get("sha256").String() != l.checksum() || !sameCounts(e.Get("counts"), l.counts) { + return nil, fmt.Errorf("invalid footer checksum or counts") + } + footer = true + continue + default: + return nil, fmt.Errorf("unknown entry type at byte %d", l.Bytes) + } + l.line(line) + } + return l, nil +} +func sameCounts(r ojson.Result, c counts) bool { + return r.Get("calls").Int() == c.Calls && r.Get("blobs").Int() == c.Blobs && r.Get("unique_blob_bytes").Int() == c.Unique && r.Get("referenced_blob_bytes").Int() == c.Referenced && r.Get("jsonl_bytes").Int() == c.Bytes +} +func recoverSegment(path, date string) (*segment, error) { + f, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + return nil, err + } + l, err := scan(f, date, false, nil) + if err == nil { + err = f.Truncate(l.Bytes) + } + if err == nil { + _, err = f.Seek(0, io.SeekEnd) + } + if err != nil { + closeLogged(f) + return nil, err + } + s := &segment{l, path, f} + if l.Bytes == 0 { + l.date = date + d := entry("header") + _ = d.Set("schema_version", 1) + _ = d.Set("date", date) + if err = s.write(d.Bytes()); err != nil { + closeLogged(f) + return nil, err + } + } + return s, nil +} + +type Manifest struct { + File string `json:"file"` + Date string `json:"date"` + Counts counts `json:"counts"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + SourceSHA256 string `json:"source_sha256"` + UploadedTo string `json:"uploaded_to,omitempty"` +} + +func (m Manifest) bytes() []byte { + d := object() + _ = d.Set("manifest", m) + return append([]byte(d.Get("manifest").Raw), '\n') +} +func fileHash(path string) (string, int64, error) { + f, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer closeLogged(f) + h := sha256.New() + n, err := io.Copy(h, f) + return fmt.Sprintf("%x", h.Sum(nil)), n, err +} + +func atomicFile(path string, write func(*os.File) error) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".trace-tmp-*") + if err != nil { + return err + } + defer func() { + if e := os.Remove(tmp.Name()); e != nil && !errors.Is(e, os.ErrNotExist) { + log.WithError(e).Warn("trace archive temporary file cleanup") + } + }() + err = write(tmp) + if err == nil { + err = tmp.Sync() + } + if e := tmp.Close(); err == nil { + err = e + } + if err != nil { + return err + } + if err = os.Rename(tmp.Name(), path); err != nil { + return err + } + return syncDirectory(filepath.Dir(path)) +} +func syncDirectory(dir string) error { + if runtime.GOOS == "windows" { + return nil + } + f, err := os.Open(dir) + if err != nil { + return err + } + defer closeLogged(f) + return f.Sync() +} +func writeManifest(dir string, m Manifest) error { + return atomicFile(filepath.Join(dir, m.File+".manifest.json"), func(f *os.File) error { _, err := f.Write(m.bytes()); return err }) +} + +func (s *segment) seal(cfg config.TraceArchiveConfig) error { + if err := s.f.Sync(); err != nil { + return err + } + target := strings.TrimSuffix(s.path, ".live") + ".jsonl.zst" + var keys []age.Recipient + for _, key := range cfg.Recipients { + k, err := age.ParseX25519Recipient(key) + if err != nil { + return fmt.Errorf("invalid archive recipient") + } + keys = append(keys, k) + } + if len(keys) > 0 { + target += ".age" + } else { + log.Warn("sealing plaintext trace archive: no age recipients configured") + } + footer := entry("footer") + _ = footer.Set("counts", s.counts) + _ = footer.Set("sha256", s.checksum()) + if err := atomicFile(target, func(f *os.File) error { + var out io.Writer = f + var enc io.WriteCloser + if len(keys) > 0 { + var err error + enc, err = age.Encrypt(f, keys...) + if err != nil { + return err + } + out = enc + } + z, err := zstd.NewWriter(out, zstd.WithEncoderConcurrency(1)) + if err != nil { + return err + } + if _, err = s.f.Seek(0, io.SeekStart); err == nil { + _, err = io.Copy(z, s.f) + } + if err == nil { + _, err = z.Write(append(footer.Bytes(), '\n')) + } + if e := z.Close(); err == nil { + err = e + } + if enc != nil { + if e := enc.Close(); err == nil { + err = e + } + } + return err + }); err != nil { + return err + } + h, size, err := fileHash(target) + if err != nil { + return err + } + m := Manifest{File: filepath.Base(target), Date: s.date, Counts: s.counts, Size: size, SHA256: h, SourceSHA256: s.checksum()} + if err = writeManifest(filepath.Dir(target), m); err != nil { + return err + } + if err = s.f.Close(); err != nil { + return err + } + s.f = nil + return os.Remove(s.path) +} + +func listManifests(dir string) ([]Manifest, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var out []Manifest + for _, e := range entries { + file, ok := strings.CutSuffix(e.Name(), ".manifest.json") + if !ok { + continue + } + match := segmentName.FindStringSubmatch(file) + if match == nil || match[3] == "live" { + continue + } + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, err + } + r := ojson.ParseBytes(b) + if !ojson.ValidBytes(b) || r.Get("file").String() != file || r.Get("date").String() != match[1] || len(r.Get("sha256").String()) != 64 { + return nil, fmt.Errorf("invalid manifest %s", e.Name()) + } + c := r.Get("counts") + m := Manifest{File: file, Date: match[1], Counts: counts{c.Get("calls").Int(), c.Get("blobs").Int(), c.Get("unique_blob_bytes").Int(), c.Get("referenced_blob_bytes").Int(), c.Get("jsonl_bytes").Int()}, Size: r.Get("size").Int(), SHA256: r.Get("sha256").String(), SourceSHA256: r.Get("source_sha256").String(), UploadedTo: r.Get("uploaded_to").String()} + out = append(out, m) + } + return out, nil +} + +func validDate(s string) bool { + _, err := time.Parse(time.DateOnly, s) + return err == nil && len(s) == len(time.DateOnly) +} diff --git a/internal/tracearchive/upload.go b/internal/tracearchive/upload.go new file mode 100644 index 00000000000..b19fec52f43 --- /dev/null +++ b/internal/tracearchive/upload.go @@ -0,0 +1,230 @@ +package tracearchive + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +func (w *worker) maintain(ctx context.Context) { + defer close(w.maintenanceDone) + delay := time.Minute + for { + err := w.maintenance(ctx) + w.statusMu.Lock() + if err != nil { + w.lastUpload = err.Error() + } else { + w.lastUpload = "" + } + w.statusMu.Unlock() + if ctx.Err() != nil { + return + } + if err != nil { + log.WithError(err).Warn("trace archive maintenance failed") + } else { + delay = time.Minute + } + t := time.NewTimer(delay) + if err != nil { + delay = min(2*delay, time.Hour) + } + select { + case <-ctx.Done(): + t.Stop() + return + case <-w.wake: + t.Stop() + case <-t.C: + } + } +} +func (w *worker) maintenance(ctx context.Context) error { + if err := recoverEvictions(w.cfg.Dir); err != nil { + return err + } + ms, err := listManifests(w.cfg.Dir) + if err != nil { + return err + } + var first error + for i, m := range ms { + if ctx.Err() != nil { + return ctx.Err() + } + if w.cfg.Upload.Enabled && m.UploadedTo != w.cfg.Upload.Remote { + if err = w.upload(ctx, &m); err != nil { + first = err + break + } + ms[i] = m + } + } + if err = w.retain(ms); err != nil { + return errors.Join(first, err) + } + return first +} + +func (w *worker) upload(ctx context.Context, m *Manifest) error { + cfg := w.cfg.Upload + bin := cfg.Binary + if bin == "" { + bin = "rclone" + } + copyVerified := func(local, name, hash string, size int64) error { + remote := strings.TrimRight(cfg.Remote, "/") + "/" + name + if err := exec.CommandContext(ctx, bin, "copyto", "--", local, remote).Run(); err != nil { + return fmt.Errorf("rclone copyto %s: %w", name, err) + } + cmd := exec.CommandContext(ctx, bin, "cat", "--", remote) + h := sha256.New() + count := &countWriter{w: h} + cmd.Stdout = count + if err := cmd.Run(); err != nil { + return fmt.Errorf("rclone verify %s: %w", name, err) + } + if count.n != size || fmt.Sprintf("%x", h.Sum(nil)) != hash { + return fmt.Errorf("rclone verification mismatch for %s", name) + } + return nil + } + if err := copyVerified(filepath.Join(w.cfg.Dir, m.File), m.File, m.SHA256, m.Size); err != nil { + return err + } + confirmed := *m + confirmed.UploadedTo = cfg.Remote + raw := confirmed.bytes() + tmp, err := os.CreateTemp(w.cfg.Dir, ".trace-tmp-*") + if err != nil { + return err + } + defer func() { + if e := os.Remove(tmp.Name()); e != nil { + log.WithError(e).Warn("trace manifest upload cleanup") + } + }() + _, err = tmp.Write(raw) + if e := tmp.Close(); err == nil { + err = e + } + if err != nil { + return err + } + if err = copyVerified(tmp.Name(), m.File+".manifest.json", digest(raw), int64(len(raw))); err != nil { + return err + } + if err = writeManifest(w.cfg.Dir, confirmed); err == nil { + *m = confirmed + } + return err +} + +type countWriter struct { + w io.Writer + n int64 +} + +func (c *countWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +func recoverEvictions(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + for _, e := range entries { + name, ok := strings.CutSuffix(e.Name(), ".evicting") + if !ok { + continue + } + m := segmentName.FindStringSubmatch(name) + if m == nil || m[3] == "live" { + continue + } + path := filepath.Join(dir, name) + _, err = os.Stat(path + ".manifest.json") + switch { + case err == nil: + if _, exists := os.Lstat(path); !errors.Is(exists, os.ErrNotExist) { + return fmt.Errorf("cannot restore evicting segment %s", name) + } + err = os.Rename(path+".evicting", path) + case errors.Is(err, os.ErrNotExist): + err = os.Remove(path + ".evicting") + } + if err != nil { + return err + } + } + return nil +} + +func (w *worker) retain(ms []Manifest) error { + limit := w.cfg.MaxLocalSizeMB << 20 + if limit == 0 { + return nil + } + entries, err := os.ReadDir(w.cfg.Dir) + if err != nil { + return err + } + var total int64 + for _, e := range entries { + if segmentName.MatchString(strings.TrimSuffix(e.Name(), ".manifest.json")) { + i, err := e.Info() + if err != nil { + return err + } + total += i.Size() + } + } + for _, m := range ms { + if total <= limit { + break + } + if w.cfg.Upload.Enabled && m.UploadedTo != w.cfg.Upload.Remote { + continue + } + path := filepath.Join(w.cfg.Dir, m.File) + manifest := path + ".manifest.json" + sidecar, err := os.Stat(manifest) + if err != nil { + return err + } + // Keep the old data recoverable until the manifest removal commits. + if err = os.Rename(path, path+".evicting"); err != nil { + return err + } + if err = syncDirectory(w.cfg.Dir); err != nil { + return err + } + if err = os.Remove(manifest); err != nil { + return err + } + if err = syncDirectory(w.cfg.Dir); err != nil { + return err + } + if err = os.Remove(path + ".evicting"); err != nil { + return err + } + total -= m.Size + sidecar.Size() + } + if total > limit { + log.WithField("bytes", total).Warn("trace archive exceeds local limit; remaining files are live or await verified upload") + } + return nil +} diff --git a/sdk/api/handlers/handlers.go b/sdk/api/handlers/handlers.go index b0e83ca2bba..853871803db 100644 --- a/sdk/api/handlers/handlers.go +++ b/sdk/api/handlers/handlers.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gin-gonic/gin" "github.com/gorilla/websocket" @@ -645,6 +647,7 @@ func (h *BaseAPIHandler) GetContextWithCancel(handler interfaces.APIHandler, c * } parentCtx = httptransport.InheritDecorator(parentCtx, requestCtx) parentCtx = logging.InheritDeferredCapture(parentCtx, requestCtx) + parentCtx = tracearchive.Inherit(parentCtx, requestCtx) } if c != nil && logging.DeferredAPICaptureFromContext(parentCtx) == nil && !logging.DeferredAPIRequestCaptureDisabled(parentCtx) { if value, exists := c.Get(logging.DeferredAPIResponseContextKey); exists { diff --git a/sdk/api/handlers/openai/openai_responses_websocket.go b/sdk/api/handlers/openai/openai_responses_websocket.go index 45e81a72785..286eafc6b0e 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket.go +++ b/sdk/api/handlers/openai/openai_responses_websocket.go @@ -15,6 +15,8 @@ import ( "time" "unicode/utf8" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" + "github.com/MoeForge/moego/ojson" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -953,6 +955,7 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { var completedResponseID string var completedPendingToolCallIDs []string var forwardErrMsg *interfaces.ErrorMessage + archiveCtx, archiveTurn := tracearchive.Begin(c.Request.Context(), true, payload) for { lastAttemptedAuthID = pinnedAuthID @@ -965,6 +968,7 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { turnStartEpoch := inputEpoch.Load() var cliCancel handlers.APIHandlerCancelFunc cliCtx, cliCancel = h.GetContextWithCancel(h, c, executionParent) + cliCtx = tracearchive.Inherit(cliCtx, archiveCtx) cliCtx = cliproxyexecutor.WithDownstreamWebsocket(cliCtx) cliCtx = cliproxyexecutor.WithLocalWebsocketTurnTracker(cliCtx) cliCtx = cliproxyexecutor.WithWebsocketInputEpoch(cliCtx, &inputEpoch, msgEpoch) @@ -1118,6 +1122,13 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { }, ) cliCancel(nil) + if errForward != nil { + archiveTurn.Finish(http.StatusBadGateway) + } else if forwardErrMsg == nil { + archiveTurn.Finish(http.StatusOK) + } else if !failoverInPlace || cliproxyexecutor.IsUpstreamWebsocketTurnError(forwardErrMsg.Error) { + archiveTurn.Finish(forwardErrMsg.StatusCode) + } if errForward != nil { wsTerminateErr = errForward switch { diff --git a/sdk/cliproxy/usage/manager.go b/sdk/cliproxy/usage/manager.go index dc7eea9125f..839f552e781 100644 --- a/sdk/cliproxy/usage/manager.go +++ b/sdk/cliproxy/usage/manager.go @@ -723,6 +723,14 @@ func (m *Manager) RegisterNamed(name string, plugin Plugin) { m.pluginsMu.Unlock() } +// WithRecordObserver attaches a request-local observer for metadata that must +// be joined before an HTTP handler or websocket turn completes. +func WithRecordObserver(ctx context.Context, observe func(Record)) context.Context { + return context.WithValue(ctx, recordObserverKey{}, observe) +} + +type recordObserverKey struct{} + // Publish enqueues a usage record for processing. If no plugin is registered // the record will be discarded downstream. func (m *Manager) Publish(ctx context.Context, record Record) { @@ -769,6 +777,9 @@ func (m *Manager) Publish(ctx context.Context, record Record) { } } ctx = WithCapturedIdentity(ctx, record.APIKey, record.AccessProvider, record.IsNativeKey) + if observe, ok := ctx.Value(recordObserverKey{}).(func(Record)); ok && observe != nil { + observe(record) + } // ensure worker is running even if Start was not called explicitly m.Start(context.Background()) m.mu.Lock() diff --git a/web/src/features/config/components/sections/SectionLogging.tsx b/web/src/features/config/components/sections/SectionLogging.tsx index cb3a052677d..00192de33b8 100644 --- a/web/src/features/config/components/sections/SectionLogging.tsx +++ b/web/src/features/config/components/sections/SectionLogging.tsx @@ -1,10 +1,59 @@ +import { useEffect, useState } from 'react'; import { useTranslation } from 'react-i18next'; +import { apiClient } from '@/services/api/client'; import type { ConfigSectionProps } from '../../types'; import { getValidationMessage } from '../blocks/shared'; import { SettingList, TextSetting, ToggleSetting } from '../fields/FieldPrimitives'; import { SettingsSection } from '../SettingsSection'; -/** Logging and usage: debug output, log files, request logs and usage statistics. */ +type ArchiveStatus = { + today_calls: number; + queue_depth: number; + dropped: number; + last_error?: string; + last_upload_error?: string; +}; + +function ArchiveStatusLine() { + const { t } = useTranslation(); + const [status, setStatus] = useState(); + useEffect(() => { + const controller = new AbortController(); + let timer: ReturnType; + async function refresh() { + try { + const next = await apiClient.get('/trace-archive/status', { + signal: controller.signal, + }); + if (!controller.signal.aborted) setStatus(next); + } catch { + if (!controller.signal.aborted) setStatus('unavailable'); + } + if (!controller.signal.aborted) timer = setTimeout(refresh, 30000); + } + void refresh(); + return () => { + controller.abort(); + clearTimeout(timer); + }; + }, []); + if (!status) return null; + const key = 'config_management.visual.sections.system.'; + if (status === 'unavailable') return
{t(key + 'trace_archive_status_failed')}
; + return ( +
+ {t(key + 'trace_archive_status', { + calls: status.today_calls, + queued: status.queue_depth, + dropped: status.dropped, + })} + {status.last_error &&
{status.last_error}
} + {status.last_upload_error &&
{status.last_upload_error}
} +
+ ); +} + +/** Logging and usage settings. */ export function SectionLogging({ values, validationErrors, @@ -66,6 +115,30 @@ export function SectionLogging({ /> + + + {t(key('trace_archive_desc'))} + + + } + checked={values.traceArchiveEnabled} + disabled={disabled} + onChange={(traceArchiveEnabled) => onChange({ traceArchiveEnabled })} + /> + onChange({ traceArchiveDir })} + /> + + = { payloadOverrideRawRules: ['payloadOverrideRawRules'], payloadFilterRules: ['payloadFilterRules'], // ── quota history ───────────────────────────────────────────────────────── + traceArchiveEnabled: ['traceArchiveEnabled'], + traceArchiveDir: ['traceArchiveDir'], requestLog: ['requestLog'], quotaHistoryEnabled: ['quotaHistoryEnabled'], quotaHistoryInterval: ['quotaHistoryInterval'], diff --git a/web/src/features/config/searchIndex.ts b/web/src/features/config/searchIndex.ts index 9c8961e6d51..5583387b2f6 100644 --- a/web/src/features/config/searchIndex.ts +++ b/web/src/features/config/searchIndex.ts @@ -725,6 +725,20 @@ export const CONFIG_FIELD_SEARCH_INDEX: ConfigFieldSearchEntry[] = [ hintKey: L('sections.payload.filter_rules_desc'), keywords: ['payload', 'rule', 'filter'], }, + { + fieldId: 'traceArchiveEnabled', + sectionId: 'logging', + labelKey: L('sections.system.trace_archive'), + yamlKeys: ['trace-archive', 'enabled'], + keywords: ['trace', 'archive'], + }, + { + fieldId: 'traceArchiveDir', + sectionId: 'logging', + labelKey: L('sections.system.trace_archive_dir'), + yamlKeys: ['trace-archive', 'dir'], + keywords: ['trace', 'archive', 'folder'], + }, { fieldId: 'requestLog', sectionId: 'logging', diff --git a/web/src/features/config/visualConfigAdditions.ts b/web/src/features/config/visualConfigAdditions.ts index 20f4d4b17bf..f85f0f68468 100644 --- a/web/src/features/config/visualConfigAdditions.ts +++ b/web/src/features/config/visualConfigAdditions.ts @@ -138,6 +138,8 @@ export const ADDITION_FIELDS = [ }, // Backend: internal/config/quota_history.go and internal/quotahistory/settings.go. { key: 'requestLog', path: 'observability.logs.request-log'.split('.'), kind: 'boolean' }, + { key: 'traceArchiveEnabled', path: 'trace-archive.enabled'.split('.'), kind: 'boolean' }, + { key: 'traceArchiveDir', path: 'trace-archive.dir'.split('.'), kind: 'string' }, { key: 'quotaHistoryEnabled', path: 'quota-history.enabled'.split('.'), kind: 'boolean' }, { key: 'quotaHistoryInterval', path: 'quota-history.interval'.split('.'), kind: 'string' }, { key: 'quotaHistoryRetention', path: 'quota-history.retention'.split('.'), kind: 'string' }, diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index b0497ee00f2..cd6f0662a72 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -1412,6 +1412,13 @@ "antigravity_signature_strict": "Strict bypass signature validation", "antigravity_signature_strict_desc": "Only applies when the signature cache is disabled; validates the full Claude protobuf tree", "request_log": "Request logs", + "trace_archive": "Trace archive", + "trace_archive_record": "Archive model calls", + "trace_archive_desc": "Save prompts, tool calls and responses. Set age recipients in YAML to encrypt sealed archives.", + "trace_archive_dir": "Archive directory", + "trace_archive_dir_hint": "Defaults to the traces folder in your auth directory.", + "trace_archive_status": "{{calls}} calls today · {{queued}} queued · {{dropped}} dropped", + "trace_archive_status_failed": "Archive status unavailable", "request_log_desc": "Record full request and response bodies. Use only while debugging; logs can contain prompts.", "logs_max_size_desc": "Oldest files are removed above this total. 0 keeps every file." }, diff --git a/web/src/types/visualConfig.ts b/web/src/types/visualConfig.ts index 450c4d418a6..849d406eeba 100644 --- a/web/src/types/visualConfig.ts +++ b/web/src/types/visualConfig.ts @@ -163,6 +163,8 @@ export type VisualConfigValues = { codexLiveMediaRelayICEServers: CodexLiveICEServerDraft[]; /** quota-history.* (t3code fork recorder). */ /** observability.logs.request-log (applies without restart). */ + traceArchiveEnabled: boolean; + traceArchiveDir: string; requestLog: boolean; quotaHistoryEnabled: boolean; quotaHistoryInterval: string; @@ -271,6 +273,8 @@ export const DEFAULT_VISUAL_VALUES: VisualConfigValues = { codexLiveMediaRelayUDPPortMin: '', codexLiveMediaRelayUDPPortMax: '', codexLiveMediaRelayICEServers: [], + traceArchiveEnabled: false, + traceArchiveDir: '', requestLog: false, quotaHistoryEnabled: false, quotaHistoryInterval: '', From 48a5fc395f83ea20e685b3907288bc88dc2ad2d3 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 20 Nov 2026 08:48:33 +0900 Subject: [PATCH 2/3] Secure trace archives and preserve per-turn capture outcomes --- config.example.yaml | 8 +- config.hub.example.yaml | 8 +- docs/trace-archive.md | 12 +- internal/api/handlers/management/config_v8.go | 7 +- internal/api/server_trace_archive.go | 2 + internal/api/server_trace_archive_test.go | 68 ++++++-- internal/config/trace_archive.go | 8 +- internal/doctor/config.go | 2 +- .../executor/codex_executor_compact_test.go | 17 ++ .../codex_executor_execute_usage_test.go | 7 + .../executor/codex_websockets_duplex.go | 160 +++++++++++++----- ..._websockets_duplex_initial_failure_test.go | 41 +++++ .../executor/codex_websockets_duplex_test.go | 38 ++++- .../runtime/executor/helps/logging_helpers.go | 6 +- .../runtime/executor/helps/usage_helpers.go | 3 +- internal/tracearchive/archive_test.go | 150 +++++++++++++++- internal/tracearchive/capture.go | 37 +++- internal/tracearchive/json.go | 3 +- internal/tracearchive/recorder.go | 33 +++- internal/tracearchive/response.go | 40 ++++- internal/tracearchive/segment.go | 36 ++-- internal/tracearchive/upload.go | 10 +- .../openai/openai_responses_websocket.go | 7 +- .../openai_responses_websocket_forward.go | 5 + sdk/cliproxy/usage/manager.go | 3 + web/src/i18n/locales/en.json | 4 +- 26 files changed, 589 insertions(+), 126 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 0982cedbde2..4a3cf4dbc8a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -9,12 +9,14 @@ # The compatibility notes at the end show old spellings for existing deployments. config-version: 8 -# Durable model-call history. Live files contain private conversation data. +# Durable model-call history. The current live segment stays local in plaintext (0600 on POSIX). # See docs/trace-archive.md for encryption, recovery, and export. # trace-archive: # enabled: false -# dir: "" # default: /traces -# recipients: [] # age1... X25519 public keys +# dir: "" # default: -traces (sibling directory) +# recipients: [] # required when enabled; use the age1... public key +# Generate on the reader machine: age-keygen -o trace-identity.txt +# Print the public recipient: age-keygen -y trace-identity.txt # queue-size: 32 # max-local-size-mb: 0 # zero disables local eviction # exclude-endpoints: [] diff --git a/config.hub.example.yaml b/config.hub.example.yaml index 7b1769e1ae3..ffaf26c8d38 100644 --- a/config.hub.example.yaml +++ b/config.hub.example.yaml @@ -5,12 +5,14 @@ # documented in config.example.yaml. config-version: 8 -# Durable model-call history. Live files contain private conversation data. +# Durable model-call history. The current live segment stays local in plaintext (0600 on POSIX). # See docs/trace-archive.md for encryption, recovery, and export. # trace-archive: # enabled: false -# dir: "" # default: /traces -# recipients: [] # age1... X25519 public keys +# dir: "" # default: -traces (sibling directory) +# recipients: [] # required when enabled; use the age1... public key +# Generate on the reader machine: age-keygen -o trace-identity.txt +# Print the public recipient: age-keygen -y trace-identity.txt # queue-size: 32 # max-local-size-mb: 0 # zero disables local eviction # exclude-endpoints: [] diff --git a/docs/trace-archive.md b/docs/trace-archive.md index c3ed9d8b6a5..6badd2347c1 100644 --- a/docs/trace-archive.md +++ b/docs/trace-archive.md @@ -4,7 +4,7 @@ Enable `trace-archive.enabled` to keep model calls for evaluation, replay or tra ## Privacy and keys -A trace includes instructions, conversation history, tool arguments, tool results, generated content and whatever secrets a user pasted into a session. The live file is plaintext. Use a private directory on an encrypted volume, restrict its OS permissions to the hub account, and protect exports just as carefully. New directories use mode 0700 and new files use 0600 on POSIX. Set Windows ACLs on the parent directory. +A trace includes instructions, conversation history, tool arguments, tool results, generated content and whatever secrets a user pasted into a session. The live segment for the current UTC day stays local in plaintext (mode 0600 on POSIX) until it seals. Use a private directory on an encrypted volume, restrict its OS permissions to the hub account, and protect exports just as carefully. New directories use mode 0700 and new files use 0600 on POSIX. Set Windows ACLs on the parent directory. Generate an age identity on the machine where you will read the archives. Keep the private identity there. Back it up somewhere other than the upload destination. Losing every matching identity makes a sealed encrypted segment unrecoverable. @@ -13,7 +13,7 @@ age-keygen -o trace-identity.txt age-keygen -y trace-identity.txt ``` -Copy only the printed `age1...` public recipient to the hub. Any configured recipient can decrypt. A segment uses the recipients configured when it seals. A hub with public keys alone can read its live file and cannot decrypt sealed history. Omitting recipients produces an unencrypted zstd file and a warning at seal time. +Copy only the printed `age1...` public recipient to the hub. Any configured recipient can decrypt. A segment uses the recipients configured when it seals. A hub with public keys alone can read its live file and cannot decrypt sealed history. Enabling the archive requires at least one valid age X25519 recipient. Every newly sealed segment is encrypted. ## Configuration and Google Drive @@ -37,7 +37,7 @@ trace-archive: 3. Apply the config and make a test call. Send an authenticated POST to `/v8/management/trace-archive/seal-now`. Check `GET /v8/management/trace-archive/status` for the segment, verified destination, dropped count and errors. 4. Test recovery on the identity machine. Copy a segment and its matching manifest into one directory, then run `traces verify` and `traces export`. -The default directory is `traces` below the resolved auth directory. Filters are exact client model names and URL paths. Queue size bounds concurrent captures plus queued calls. A call exceeding the available slots, 128 MiB per call, 256 MiB of aggregate capture buffers, or 256 levels of JSON nesting is dropped whole. Disk, compression, reconstruction, encryption and networking run outside client handlers. Counters for dropped calls reset when the hub restarts. The local limit is in MiB and zero means unlimited. Retention removes the oldest sealed pairs first. With uploads enabled, only a pair verified at the configured destination qualifies. The live segment always stays on disk. +The default directory is a sibling of the resolved auth directory, with `-traces` appended to its name (for example `~/.t3code-proxy-traces`). An explicit `trace-archive.dir` takes precedence. To continue using a previous archive location, set `dir` to that path. Startup renames legacy `.manifest.json` sidecars there to `.manifest`. Historical plaintext segments remain readable locally and are ineligible for upload. Filters are exact client model names and URL paths. Queue size bounds concurrent captures plus queued calls. A call exceeding the available slots, 128 MiB per call, 256 MiB of aggregate capture buffers, or 256 levels of JSON nesting is dropped whole. Disk, compression, reconstruction, encryption and networking run outside client handlers. Counters for dropped calls reset when the hub restarts. The local limit is in MiB and zero means unlimited. Retention removes the oldest sealed pairs first. With uploads enabled, only a pair verified at the configured destination qualifies. The live segment always stays on disk. Both files are sent with `rclone copyto`. The hub streams each back with `rclone cat` to check the exact byte length and SHA-256 even on remotes that expose only MD5. This uses bandwidth in both directions. A verified pair has the destination in local and remote manifest field `uploaded_to`. Errors retry in the background with exponential backoff up to one hour. Uploads are cancellable at shutdown. @@ -48,18 +48,18 @@ age -d -i trace-identity.txt traces-2026-11-20-123.jsonl.zst.age | zstd -dc > re jq -c 'select(.type == "call") | .call.response' restored.jsonl ``` -Schema version 1 uses UTF-8 JSON objects separated by LF, one object per line, including the final LF. Dates are UTC. File names are `traces-YYYY-MM-DD[-n].live` while open and `traces-YYYY-MM-DD[-n].jsonl.zst[.age]` once sealed. The writer uses a nanosecond suffix to distinguish same-day segments. The date is the recording day. A new day or a manual seal starts an independent blob dictionary. A current-day live file can survive a clean shutdown. Recovery truncates only the unterminated final line, rejects a corrupt complete line and seals stale live files. +Schema version 1 uses UTF-8 JSON objects separated by LF, one object per line, including the final LF. Dates are UTC. File names are `traces-YYYY-MM-DD[-n].live` while open and `traces-YYYY-MM-DD[-n].jsonl.zst.age` once sealed. The writer uses a nanosecond suffix to distinguish same-day segments. The date is the recording day. A new day or a manual seal starts an independent blob dictionary. A current-day live file can survive a clean shutdown. Recovery truncates only the unterminated final line, rejects a corrupt complete line and seals stale live files. Entries appear in this order: * A header has `type: "header"`, integer `schema_version: 1` and `date`. * A blob has `type: "blob"`, `sha256` (lowercase hex) and `value` (any JSON value). Hash the UTF-8 canonical JSON value. Objects sort keys by their UTF-8 byte strings recursively, arrays retain order, and separators have no whitespace. Strings use UTF-8 in double quotes. Escape quotes, backslashes and the five short control escapes. Encode other ASCII control characters, `<`, `>` and `&` as lowercase `\u00xx`, and U+2028/U+2029 as `\u2028`/`\u2029`. Numeric literals retain their original spelling, including large integers. Each hash appears once and before its first reference. * A call has `type: "call"` and a `call` object with its own integer `schema_version`. Its packed `client_request` and `upstream_request` each have a `body` and optional `parts`. Copy body, then insert each parts entry at the same top-level key. A string denotes one blob hash. An array denotes an ordered array of blob hashes. Empty arrays remain empty. Fields eligible for extraction are system, instructions, tools, messages, input, contents, systemInstruction and toolConfig. The `base64` alternative holds a non-JSON request as standard padded Base64. Missing payload fields mean that side was unobserved. -* The call's `response` is the reconstructed upstream object. Claude thinking, signatures, redacted blocks, tools, citations and stop information stay in provider format. Responses items retain opaque encrypted content and summaries. Chat chunks become choices with complete messages and tool calls. For HTTP calls, a differing final client result is stored as `client_response`, or `client_raw_base64` if it cannot be reconstructed. The client-facing body supplies a fallback when upstream capture is unavailable. When reconstruction fails, `raw_frames_base64` contains ordered standard Base64 snapshots of executor chunks (scanner lines, HTTP bodies or WebSocket messages). Base64 decoding recovers each frame independently. Network headers are omitted. Recognized credential fields inside JSON header and metadata containers are masked. +* The call's `response` is the reconstructed upstream object. Claude thinking, signatures, redacted blocks, tools, citations and stop information stay in provider format. Responses items retain opaque encrypted content and summaries. Chat chunks become choices with complete messages and tool calls. For HTTP calls, a differing final client result is stored as `client_response`, or `client_raw_base64` if it cannot be reconstructed. The client-facing body supplies a fallback when upstream capture is unavailable. When reconstruction fails, `raw_frames_base64` contains ordered standard Base64 snapshots of executor chunks (scanner lines, HTTP bodies or WebSocket messages). Base64 decoding recovers each frame independently. Network headers are omitted. MCP authorization fields and recognized credentials in JSON header and metadata containers are masked before blob hashing and storage. * Metadata includes hub version, client URL path and protocol, user agent, client name when supplied, HTTP or WS transport, stream flag, session lineage, trace and request IDs, provider, executor, non-secret auth index, requested/alias/upstream/response models, effort, tier, status, failure classification and token detail where available. Attempts are ordered by outbound send. The final upstream body is paired with the final response. Times are RFC 3339 UTC. Latency and TTFT are integer nanoseconds. `first_byte` is the first observed HTTP response byte or WS frame. `ttft_ns` uses provider token timing where available and falls back to the first observed response byte. Optional fields can be absent for early failures or providers that omit them. * A sealed segment ends with `type: "footer"`, `counts` and `sha256`. Hash the exact decompressed bytes of every preceding line **including its LF**. Counts are calls, blobs, unique_blob_bytes, referenced_blob_bytes and jsonl_bytes (before the footer). The dedup ratio divides bytes across all blob references by unique blob bytes. A live segment has no footer. -The sidecar `.manifest.json` contains file, date, counts, size (sealed bytes), sha256 (sealed file), source_sha256 (same digest as footer) and optional uploaded_to. The hidden writer lock and temporary files are local implementation details. Export needs only the segment pairs, matching identities for encrypted files and any live segment to include. +The sidecar `.manifest` contains file, date, counts, size (sealed bytes), sha256 (sealed file), source_sha256 (same digest as footer) and optional uploaded_to. The hidden writer lock and temporary files are local implementation details. Export needs only the segment pairs, matching identities for encrypted files and any live segment to include. To re-inline both requests from a decrypted segment using jq alone: diff --git a/internal/api/handlers/management/config_v8.go b/internal/api/handlers/management/config_v8.go index 55f802a35f6..58be14d3193 100644 --- a/internal/api/handlers/management/config_v8.go +++ b/internal/api/handlers/management/config_v8.go @@ -2,6 +2,7 @@ package management import ( "encoding/json" + "errors" "io" "net/http" "os" @@ -191,7 +192,11 @@ func (h *Handler) ConfigV8(c *gin.Context) { } next, err := config.ParseConfigBytes(data) if err != nil { - c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "invalid_config", "message": err.Error()}) + status := http.StatusUnprocessableEntity + if errors.Is(err, config.ErrTraceArchiveRecipient) { + status = http.StatusBadRequest + } + c.JSON(status, gin.H{"error": "invalid_config", "message": err.Error()}) return } if err = config.ValidateV8Config(data); err != nil { diff --git a/internal/api/server_trace_archive.go b/internal/api/server_trace_archive.go index 32599dbe502..027ab4c510f 100644 --- a/internal/api/server_trace_archive.go +++ b/internal/api/server_trace_archive.go @@ -7,6 +7,7 @@ import ( "github.com/gin-gonic/gin" "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/httpwire" "github.com/rune-oai/t3code-proxy/internal/logging" "github.com/rune-oai/t3code-proxy/internal/tracearchive" log "github.com/sirupsen/logrus" @@ -35,6 +36,7 @@ type traceWriter struct { } func (w traceWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } +func (w traceWriter) FlushError() error { return httpwire.FlushResponse(w.ResponseWriter) } func (w traceWriter) Write(p []byte) (int, error) { n, e := w.ResponseWriter.Write(p) w.c.ClientResponse(p[:n]) diff --git a/internal/api/server_trace_archive_test.go b/internal/api/server_trace_archive_test.go index 76a7e4991e6..9fe86c88666 100644 --- a/internal/api/server_trace_archive_test.go +++ b/internal/api/server_trace_archive_test.go @@ -3,24 +3,50 @@ package api import ( "bytes" "context" + "errors" "fmt" "io" "net/http" "net/http/httptest" "os" + "path/filepath" "reflect" "strings" "testing" + "filippo.io/age" "github.com/MoeForge/moego/ojson" + "github.com/gin-gonic/gin" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/httpwire" "github.com/rune-oai/t3code-proxy/internal/registry" "github.com/rune-oai/t3code-proxy/internal/runtime/executor" "github.com/rune-oai/t3code-proxy/internal/tracearchive" "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" ) +func testArchiveSettings(t *testing.T) (config.TraceArchiveConfig, string) { + t.Helper() + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatal(err) + } + key := filepath.Join(t.TempDir(), "identity") + if err = os.WriteFile(key, []byte(id.String()+"\n"), 0600); err != nil { + t.Fatal(err) + } + return config.TraceArchiveConfig{Enabled: true, Dir: t.TempDir(), Recipients: []string{id.Recipient().String()}}, key +} + +func TestTraceWriterForwardsFlushError(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + w := traceWriter{ResponseWriter: &compressionErrorWriter{ResponseWriter: c.Writer}} + if err := httpwire.FlushResponse(w); !errors.Is(err, io.ErrClosedPipe) { + t.Fatalf("flush error = %v", err) + } +} + func TestTraceArchiveStreamingClaudeThroughServer(t *testing.T) { const model = "archive-claude-test" const body = `{"model":"archive-claude-test","max_tokens":64,"messages":[{"role":"user","content":"Check the sky"}],"stream":true}` @@ -50,7 +76,8 @@ func TestTraceArchiveStreamingClaudeThroughServer(t *testing.T) { } })) defer upstream.Close() - cfg := &config.Config{TraceArchive: config.TraceArchiveConfig{Enabled: true, Dir: t.TempDir()}} + settings, key := testArchiveSettings(t) + cfg := &config.Config{TraceArchive: settings} cfg.APIKeys = []string{"never-archive-client-secret"} s := newTestServerWithConfig(t, cfg) defer s.archive.Close() @@ -77,7 +104,7 @@ func TestTraceArchiveStreamingClaudeThroughServer(t *testing.T) { t.Fatal(err) } var out, stderr bytes.Buffer - if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir}, &out, &stderr) != 0 { + if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir, "--identity", key}, &out, &stderr) != 0 { t.Fatal(stderr.String()) } result := ojson.ParseBytes(out.Bytes()) @@ -131,7 +158,10 @@ func TestTraceArchiveCodexWebsocketThroughServer(t *testing.T) { _, _, _ = conn.ReadMessage() })) defer upstream.Close() - cfg := &config.Config{TraceArchive: config.TraceArchiveConfig{Enabled: true, Dir: t.TempDir()}} + settings, key := testArchiveSettings(t) + cfg := &config.Config{TraceArchive: settings} + cfg.Codex.ResponseSteering = true + cfg.CodexResponseSteering = true cfg.APIKeys = []string{"test-key"} s := newTestServerWithConfig(t, cfg) defer s.archive.Close() @@ -176,10 +206,13 @@ func TestTraceArchiveCodexWebsocketThroughServer(t *testing.T) { t.Fatal(err) } var out, stderr bytes.Buffer - if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir}, &out, &stderr) != 0 { + if tracearchive.Main([]string{"export", "--dir", cfg.TraceArchive.Dir, "--identity", key}, &out, &stderr) != 0 { t.Fatal(stderr.String()) } record := ojson.ParseBytes(out.Bytes()) + if record.Get("status").Int() != 200 { + t.Fatalf("initial turn status = %s", record.Get("status")) + } if record.Get("response.output.0.encrypted_content").String() != "sealed-reasoning" || record.Get("response.output.0.summary.0.text").String() != "abbreviated" || record.Get("transport").String() != "ws" || record.Get("attempts.0.transport").String() != "ws" { t.Fatal(out.String()) } @@ -196,7 +229,7 @@ func TestTraceArchiveManagement(t *testing.T) { if err := os.WriteFile(s.configFilePath, []byte("config-version: 8\n"), 0600); err != nil { t.Fatal(err) } - request := func(method, path, body string, status int) string { + request := func(t *testing.T, method, path, body string, status int) string { t.Helper() req := httptest.NewRequest(method, path, strings.NewReader(body)) req.RemoteAddr = "127.0.0.1:1234" @@ -209,14 +242,23 @@ func TestTraceArchiveManagement(t *testing.T) { return w.Body.String() } const base = "/v8/management/trace-archive/" - request("GET", "/v0/management/trace-archive/status", "", 404) - request("GET", base+"settings", "", 200) - request("POST", base+"seal-now", "", 409) - request("PUT", base+"settings", `{"enabled":true,"recipients":["bad"]}`, 422) + request(t, "GET", "/v0/management/trace-archive/status", "", 404) + request(t, "GET", base+"settings", "", 200) + request(t, "POST", base+"seal-now", "", 409) + t.Run("RequiresX25519Recipient", func(t *testing.T) { + for _, body := range []string{`{"enabled":true}`, `{"enabled":true,"recipients":["bad"]}`} { + result := request(t, "PUT", base+"settings", body, 400) + if !strings.Contains(ojson.Get(result, "message").String(), "X25519 recipient") { + t.Fatal(result) + } + } + }) settings := ojson.NewDocString(`{"enabled":true}`) + valid, _ := testArchiveSettings(t) + _ = settings.Set("recipients", valid.Recipients) _ = settings.Set("dir", cfg.TraceArchive.Dir) - request("PUT", base+"settings", settings.String(), 200) - if !ojson.Get(request("GET", base+"settings", "", 200), "enabled").Bool() { + request(t, "PUT", base+"settings", settings.String(), 200) + if !ojson.Get(request(t, "GET", base+"settings", "", 200), "enabled").Bool() { t.Fatal("settings read missed persisted update") } // Exercise the same update callback as the file watcher. @@ -229,8 +271,8 @@ func TestTraceArchiveManagement(t *testing.T) { t.Fatal(err) } s.UpdateClients(next) - if !ojson.Get(request("GET", base+"status", "", 200), "enabled").Bool() { + if !ojson.Get(request(t, "GET", base+"status", "", 200), "enabled").Bool() { t.Fatal("watcher did not enable recorder") } - request("POST", base+"seal-now", "", 200) + request(t, "POST", base+"seal-now", "", 200) } diff --git a/internal/config/trace_archive.go b/internal/config/trace_archive.go index 847f63823cd..deb832c6794 100644 --- a/internal/config/trace_archive.go +++ b/internal/config/trace_archive.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "strings" @@ -24,13 +25,18 @@ type TraceUploadConfig struct { Remote string `yaml:"remote,omitempty" json:"remote"` } +var ErrTraceArchiveRecipient = errors.New("trace-archive: configure at least one valid age X25519 recipient in trace-archive.recipients") + func (c TraceArchiveConfig) Validate() error { if c.QueueSize < 0 || c.QueueSize > 65536 || c.MaxLocalSizeMB < 0 || c.MaxLocalSizeMB > 1<<40 { return fmt.Errorf("trace-archive: invalid queue-size or max-local-size-mb") } + if c.Enabled && len(c.Recipients) == 0 { + return ErrTraceArchiveRecipient + } for _, key := range c.Recipients { if _, err := age.ParseX25519Recipient(key); err != nil { - return fmt.Errorf("trace-archive: invalid X25519 recipient") + return ErrTraceArchiveRecipient } } if c.Upload.Enabled && (strings.TrimSpace(c.Upload.Remote) == "" || strings.HasPrefix(c.Upload.Remote, "-")) { diff --git a/internal/doctor/config.go b/internal/doctor/config.go index c0ab396a75f..e3d3d2c57ee 100644 --- a/internal/doctor/config.go +++ b/internal/doctor/config.go @@ -104,7 +104,7 @@ func checkConfig(e *env) Result { host := c.Server.Host var problems, warnings []string if c.TraceArchive.Enabled && len(c.TraceArchive.Recipients) == 0 { - warnings = append(warnings, "trace archive has no age recipients; sealed history is plaintext") + problems = append(problems, "trace archive requires an age X25519 recipient") } switch host { case "127.0.0.1", "localhost", "::1": diff --git a/internal/runtime/executor/codex_executor_compact_test.go b/internal/runtime/executor/codex_executor_compact_test.go index b6d52d716fe..21f06d675a1 100644 --- a/internal/runtime/executor/codex_executor_compact_test.go +++ b/internal/runtime/executor/codex_executor_compact_test.go @@ -1499,6 +1499,7 @@ func TestCodexExecutor_IsCompatCompactionDuplexSession_Issue7406(t *testing.T) { ctx = cliproxyexecutor.WithWebsocketInput(cliproxyexecutor.WithDownstreamWebsocket(ctx), inputCh) turn1 := []byte(`{"type":"response.create","model":"client-alias","instructions":"Keep concise.","input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"Remember ` + marker + `"}]}]}`) + ctx, initialCapture, export := beginTestDuplexArchive(t, ctx, turn1) streamRes, errStream := wsExec.ExecuteStream(ctx, auth, cliproxyexecutor.Request{ Model: "deepseek-v4.1-flash", Payload: turn1, @@ -1648,6 +1649,22 @@ func TestCodexExecutor_IsCompatCompactionDuplexSession_Issue7406(t *testing.T) { if staleErrEv.Get("status").Int() != http.StatusConflict || staleErrEv.Get("error.code").String() != "previous_response_not_found" { t.Fatalf("unexpected stale synthetic ID error: %s", staleErrEv.Raw) } + cancel() + for range streamRes.Chunks { + } + initialCapture.Finish(200) + t.Run("ArchiveHTTPCompaction", func(t *testing.T) { + for _, call := range export() { + if call.Get("client_request.instructions").String() != "INSTRUCTIONS_B" { + continue + } + if call.Get("status").Int() != 200 || call.Get("attempts.0.transport").String() != "http" || call.Get("token_usage.TotalTokens").Int() != 15 || call.Get("response.id").String() != "resp_http_sum" || call.Get("client_response.id").String() != synthRespID || call.Get("upstream_request").Raw == "" { + t.Fatal(call.Raw) + } + return + } + t.Fatal("no compaction call in archive") + }) } func TestCodexExecutor_IsCompatCompactionDuplexEOFCancelsHTTP_Issue7406(t *testing.T) { diff --git a/internal/runtime/executor/codex_executor_execute_usage_test.go b/internal/runtime/executor/codex_executor_execute_usage_test.go index a43608c395b..d1e56eae9b1 100644 --- a/internal/runtime/executor/codex_executor_execute_usage_test.go +++ b/internal/runtime/executor/codex_executor_execute_usage_test.go @@ -63,6 +63,8 @@ func TestCodexExecutorExecutePublishesMainUsageBeforeImageUsage(t *testing.T) { coreusage.RegisterNamedPlugin(t.Name(), codexResponseModelNoopUsagePlugin{}) }) ctx := coreusage.WithRequestedModelAlias(context.Background(), alias) + var primary []coreusage.Record + ctx = coreusage.WithRecordObserver(ctx, func(r coreusage.Record) { primary = append(primary, r) }) executor := NewCodexExecutor(&config.Config{}) resp, errExecute := executor.Execute(ctx, codexOAuthTestAuth(server.URL), cliproxyexecutor.Request{ Model: "gpt-5.5", @@ -84,6 +86,11 @@ func TestCodexExecutorExecutePublishesMainUsageBeforeImageUsage(t *testing.T) { } if imageUsage.name == "nonzero" { imageRecord := capture.await(t) + t.Run("PrimaryAttemptObserver", func(t *testing.T) { + if len(primary) != 1 || primary[0].RequestID != mainRecord.RequestID || primary[0].Detail.TotalTokens != wantMain.TotalTokens { + t.Fatalf("observed %d records for the primary attempt", len(primary)) + } + }) if imageRecord.Model != codexDefaultImageToolModel || imageRecord.Failed { t.Errorf("image record model = %q, failed = %t", imageRecord.Model, imageRecord.Failed) } diff --git a/internal/runtime/executor/codex_websockets_duplex.go b/internal/runtime/executor/codex_websockets_duplex.go index 00d7a27df00..9c4c85b71ce 100644 --- a/internal/runtime/executor/codex_websockets_duplex.go +++ b/internal/runtime/executor/codex_websockets_duplex.go @@ -11,12 +11,12 @@ import ( "strings" "sync" - "github.com/rune-oai/t3code-proxy/internal/tracearchive" - "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" + "github.com/rune-oai/t3code-proxy/internal/clienterror" "github.com/rune-oai/t3code-proxy/internal/runtime/executor/helps" "github.com/rune-oai/t3code-proxy/internal/thinking" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" cliproxyauth "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" cliproxyexecutor "github.com/rune-oai/t3code-proxy/sdk/cliproxy/executor" coreusage "github.com/rune-oai/t3code-proxy/sdk/cliproxy/usage" @@ -72,10 +72,16 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( pending := []*codexWebsocketPrepared{initial} // Serialize explicit creates against steering continuations. A response.created // alone does not identify whether upstream created it automatically. - type codexSteerSubmission struct{ parent, eventID string } + type codexSteerSubmission struct { + parent, eventID string + archive *tracearchive.Capture + } // successorID is set once the response that consumes the steer has started; the steer // stays outstanding until that response ends. - type codexAcceptedSteer struct{ parent, eventID, successorID string } + type codexAcceptedSteer struct { + codexSteerSubmission + successorID string + } var unacknowledgedSteers []codexSteerSubmission acceptedSteers := make(map[string]codexAcceptedSteer) current := initial @@ -112,9 +118,6 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( return } } - if p := steeringSettings[parent]; p != nil && p.archive != current.archive { - p.archive.Finish(http.StatusAccepted) - } delete(steeringSettings, parent) } removeAcceptedSteer := func(id string) { @@ -541,16 +544,16 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( metadataMu.Lock() localCompactionDeliveredAny = false metadataMu.Unlock() - compactionResp, errCompact := e.CodexExecutor.executeCodexResponsesCompaction(writerFollowupCtx, auth, nextReq, nextOpts) + turnCtx, capture := tracearchive.Begin(writerFollowupCtx, true, consumedPayload) + // Admission can drop this capture. Clear any inherited initial-turn observer. + status := http.StatusBadGateway + defer func() { capture.Finish(status) }() + compactionResp, errCompact := e.CodexExecutor.executeCodexResponsesCompaction(capture.Context(turnCtx), auth, nextReq, nextOpts) if errCompact != nil { + status = clienterror.HTTPStatusFromErrorOr(errCompact, http.StatusBadGateway) if streamCtx.Err() != nil { return false } - var statusHolder interface{ StatusCode() int } - status := http.StatusBadGateway - if errors.As(errCompact, &statusHolder) && statusHolder.StatusCode() > 0 { - status = statusHolder.StatusCode() - } if status == http.StatusUnauthorized || status == http.StatusForbidden || status == http.StatusTooManyRequests { failWithClassification(errCompact, true) return false @@ -600,12 +603,14 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( for idx, chunk := range chunks { select { case out <- cliproxyexecutor.StreamChunk{Payload: chunk}: + capture.ClientResponse(chunk) if idx == 0 { metadataMu.Lock() localCompactionDeliveredAny = true metadataMu.Unlock() } case <-streamCtx.Done(): + status = clienterror.HTTPStatusFromErrorOr(streamCtx.Err(), http.StatusBadGateway) return false } } @@ -613,6 +618,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( localCompactionDeliveredAny = false metadataMu.Unlock() consumedPayload = nil + status = http.StatusOK return true } prepared, errPrepare := e.prepareCodexWebsocketStream(streamCtx, auth, nextReq, nextOpts) @@ -762,18 +768,11 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( metadataMu.Unlock() } metadataMu.Lock() - unacknowledgedSteers = append(unacknowledgedSteers, codexSteerSubmission{parent: parent, eventID: steerEventID}) - consumedPayload = nil steerCtx, steerArchive := tracearchive.Begin(streamCtx, true, message.Payload) + unacknowledgedSteers = append(unacknowledgedSteers, codexSteerSubmission{parent: parent, eventID: steerEventID, archive: steerArchive}) + consumedPayload = nil if settings != nil { - snapshot := *settings - snapshot.archive = steerArchive - if prior := steeringSettings[parent]; prior != nil { - prior.archive.Finish(http.StatusAccepted) - } - steeringSettings[parent] = &snapshot - } else { - steerArchive.Finish(http.StatusAccepted) + steeringSettings[parent] = settings } metadataMu.Unlock() // Control frames bypass response.create translations and built-in defaults. @@ -818,6 +817,16 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( } }() go func() { + // Bootstrap rejections can still retry. The handler owns the initial + // capture across those attempts and throughout the committed turn. + finishResponse := func(owner *codexWebsocketPrepared, status int) { + owner.archive.Status(status) + if owner != initial { + owner.archive.Finish(status) + } else if !firstResponse { + owner.archive.MarkDuplexResult(status) + } + } defer close(out) defer func() { cancel() @@ -825,12 +834,19 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // releasing the execution session. No goroutine outlives its socket. e.invalidateUpstreamConnWithoutDisconnectNotify(sess, conn, "duplex_closed", nil) <-writerDone - current.archive.Finish(http.StatusBadGateway) + if current != initial { + current.archive.Finish(http.StatusBadGateway) + } for _, p := range pending { - p.archive.Finish(http.StatusBadGateway) + if p != initial { + p.archive.Finish(http.StatusBadGateway) + } + } + for _, steer := range unacknowledgedSteers { + steer.archive.Finish(http.StatusBadGateway) } - for _, p := range steeringSettings { - p.archive.Finish(http.StatusBadGateway) + for _, steer := range acceptedSteers { + steer.archive.Finish(http.StatusBadGateway) } sess.clearActive(conn, readCh) unlock() @@ -843,6 +859,35 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( return false } } + logFrame := func(capture *tracearchive.Capture, payload []byte) context.Context { + frameCtx := capture.Context(ctx) + helps.AppendCodexAPIWebsocketResponse(frameCtx, e.cfg, payload) + helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload) + return frameCtx + } + var successorCaptures []*tracearchive.Capture + recordFrame := func(owner *codexWebsocketPrepared, payload []byte) context.Context { + frameCapture := owner.archive + if owner == current { + // Several steers can be consumed by the same successor, including + // one started by an explicit create with tool results. + for _, capture := range successorCaptures { + if frameCapture == nil { + frameCapture = capture + } else { + capture.Chunk(payload) + } + } + } + return logFrame(frameCapture, payload) + } + finishSuccessorSteers := func(status int) { + for _, capture := range successorCaptures { + capture.Status(status) + capture.Finish(status) + } + successorCaptures = nil + } reporter := initialReporter outputItems := make(map[int64][]byte) var outputFallback [][]byte @@ -924,7 +969,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( connectionErr = turnErr } } - reporter.PublishFailure(ctx, connectionErr) + reporter.PublishFailure(current.archive.Context(ctx), connectionErr) send(cliproxyexecutor.StreamChunk{Err: connectionErr}) } return @@ -951,15 +996,19 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( if settings == nil { metadataMu.Unlock() connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("automatic successor has no retained parent settings")} - reporter.PublishFailure(ctx, connectionErr) + reporter.PublishFailure(current.archive.Context(ctx), connectionErr) send(cliproxyexecutor.StreamChunk{Err: connectionErr}) return } current = settings } createdResponseID := strings.Clone(ojson.GetBytesNoCopy(payload, "response.id").String()) + successorCaptures = nil for id, accepted := range acceptedSteers { if accepted.parent == parent && accepted.successorID == "" { + if accepted.archive != nil { + successorCaptures = append(successorCaptures, accepted.archive) + } accepted.successorID = createdResponseID acceptedSteers[id] = accepted } @@ -1003,9 +1052,6 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( outputFallback = nil } observeCodexTokenEvent(reporter, payload) - frameCtx := current.archive.Context(ctx) - helps.AppendCodexAPIWebsocketResponse(frameCtx, e.cfg, payload) - helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload) // Steering acknowledgements, pending notifications and failures are opaque: // preserve their IDs, input, sequence numbers and event types byte-for-byte. if strings.HasPrefix(eventType, "response.steer.") { @@ -1024,14 +1070,17 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( } return codexSteerSubmission{} } + submission := acceptedSteers[id].codexSteerSubmission switch eventType { case "response.steer.accepted": - acceptedSteers[id] = codexAcceptedSteer{parent: parent, eventID: consumeSubmission().eventID} + submission = consumeSubmission() + submission.parent = parent + acceptedSteers[id] = codexAcceptedSteer{codexSteerSubmission: submission} case "response.steer.failed": if _, accepted := acceptedSteers[id]; accepted { removeAcceptedSteer(id) } else { - consumeSubmission() + submission = consumeSubmission() } releaseSteeringSettings(parent) case "response.steer.pending": @@ -1040,6 +1089,18 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( waitingParent = parent } metadataMu.Unlock() + logFrame(submission.archive, payload) + if eventType == "response.steer.failed" { + status := int(ojson.GetBytesNoCopy(payload, "status").Int()) + if status == 0 { + status = int(ojson.GetBytesNoCopy(payload, "status_code").Int()) + } + if status < 400 || status > 599 { + status = http.StatusBadRequest + } + submission.archive.Status(status) + submission.archive.Finish(status) + } wakeWriter() if !send(cliproxyexecutor.StreamChunk{Payload: payload}) { return @@ -1061,7 +1122,10 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // Account health is independent of which queued request failed. // The conductor records the original classification without replaying // this already-started stream on another credential. - reporter.PublishFailure(ctx, credentialErr) + frameCtx := recordFrame(current, payload) + reporter.PublishFailure(frameCtx, credentialErr) + finishResponse(current, status.StatusCode()) + finishSuccessorSteers(status.StatusCode()) if send(cliproxyexecutor.StreamChunk{Payload: payload}) { send(cliproxyexecutor.StreamChunk{Err: credentialErr}) } @@ -1093,17 +1157,19 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( metadataMu.Unlock() wakeWriter() if ambiguous { + logFrame(nil, payload) // Without a response ID, assigning this failure could corrupt // either request. Preserve the event and fail the socket without // guessing a scope, replaying input, or cooling the credential. connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("cannot associate websocket failure with a response or pending create")} - reporter.PublishFailure(ctx, connectionErr) + reporter.PublishFailure(current.archive.Context(ctx), connectionErr) if send(cliproxyexecutor.StreamChunk{Payload: payload}) { send(cliproxyexecutor.StreamChunk{Err: connectionErr}) } return } } + frameCtx := recordFrame(eventPrepared, payload) restoreMultiAgent := !eventPrepared.multiAgentV2Conflict && (eventPrepared.optimizeMultiAgentV2 || sess.isMultiAgentV2Optimized(conn)) payload = helps.RestoreCodexMultiAgentV2Response(payload, restoreMultiAgent) // Parse and invalidate replay for every rejected request, using the @@ -1119,7 +1185,10 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( } if replayErr != nil { helps.RecordAPIWebsocketError(ctx, e.cfg, "replay_clear_error", replayErr) - eventReporter.PublishFailure(ctx, replayErr) + eventReporter.PublishFailure(frameCtx, replayErr) + if eventPrepared != initial { + eventPrepared.archive.Finish(http.StatusBadGateway) + } send(cliproxyexecutor.StreamChunk{Err: replayErr}) return } @@ -1127,11 +1196,9 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( var status interface{ StatusCode() int } terminalBusinessRejection = errors.As(terminalErr, &status) && status.StatusCode() == http.StatusBadRequest helps.RecordAPIWebsocketError(ctx, e.cfg, "upstream_error", terminalErr) - eventReporter.PublishFailure(eventPrepared.archive.Context(ctx), terminalErr) - if eventPrepared != current { - eventPrepared.archive.Chunk(payload) - } - eventPrepared.archive.Finish(http.StatusBadGateway) + eventReporter.PublishFailure(frameCtx, terminalErr) + failureStatus := clienterror.HTTPStatusFromErrorOr(terminalErr, http.StatusBadGateway) + finishResponse(eventPrepared, failureStatus) if isInitialResponse { send(cliproxyexecutor.StreamChunk{Err: terminalErr}) return @@ -1177,12 +1244,19 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( } else { reporter.EnsurePublished(frameCtx) } - current.archive.Finish(http.StatusOK) + finishResponse(current, http.StatusOK) publishCodexImageToolUsageWithOutputItems(ctx, reporter, current.clientBody, payload, outputItems, outputFallback) if e.duplexStateObserver != nil { e.duplexStateObserver("terminal_pending", false) } } + if eventPrepared == current && isCodexTerminalEventType(eventType) { + status := http.StatusOK + if terminalErr != nil { + status = clienterror.HTTPStatusFromErrorOr(terminalErr, http.StatusBadGateway) + } + finishSuccessorSteers(status) + } if !send(cliproxyexecutor.StreamChunk{Payload: helps.EnsureResponsesUsageDetails(payload), Incomplete: eventType == "response.incomplete"}) { return } diff --git a/internal/runtime/executor/codex_websockets_duplex_initial_failure_test.go b/internal/runtime/executor/codex_websockets_duplex_initial_failure_test.go index 6d13895e558..5ee1eb16b67 100644 --- a/internal/runtime/executor/codex_websockets_duplex_initial_failure_test.go +++ b/internal/runtime/executor/codex_websockets_duplex_initial_failure_test.go @@ -1,24 +1,57 @@ package executor import ( + "bytes" "context" "errors" "fmt" "net/http" "net/http/httptest" + "strings" "sync/atomic" "testing" "time" + "filippo.io/age" "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/config" "github.com/rune-oai/t3code-proxy/internal/registry" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" auth "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" core "github.com/rune-oai/t3code-proxy/sdk/cliproxy/executor" translator "github.com/rune-oai/t3code-proxy/sdk/translator" ) +// export drains the recorder and reads the live calls after the executor finishes. +func beginTestDuplexArchive(t *testing.T, ctx context.Context, body []byte) (context.Context, *tracearchive.Capture, func() []ojson.Result) { + t.Helper() + dir := t.TempDir() + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatal(err) + } + r := new(tracearchive.Recorder) + if err = r.Apply(&config.Config{TraceArchive: config.TraceArchiveConfig{Dir: dir, Enabled: true, Recipients: []string{id.Recipient().String()}}}); err != nil { + t.Fatal(err) + } + t.Cleanup(r.Close) + req := httptest.NewRequestWithContext(ctx, "GET", "/v1/responses", nil) + ctx, c := tracearchive.Begin(r.Bind(req, "openai-responses", "test"), true, body) + return ctx, c, func() []ojson.Result { + r.Close() + var out, stderr bytes.Buffer + if tracearchive.Main([]string{"export", "--dir", dir}, &out, &stderr) != 0 { + t.Fatal(stderr.String()) + } + var calls []ojson.Result + for line := range strings.Lines(out.String()) { + calls = append(calls, ojson.Parse(line)) + } + return calls + } +} + func TestCodexDuplexInitialFailure(t *testing.T) { for _, tc := range []struct { name, payload string @@ -137,6 +170,7 @@ func TestCodexDuplexInitialFailure(t *testing.T) { t.Fatal(err) } } + ctx, initialCapture, export := beginTestDuplexArchive(t, ctx, req.Payload) before := time.Now() result, err := manager.ExecuteStream(ctx, []string{"codex"}, req, opts) if err != nil { @@ -154,6 +188,13 @@ func TestCodexDuplexInitialFailure(t *testing.T) { t.Fatalf("rejected account payload escaped bootstrap: %s", chunk.Payload) } } + initialCapture.Finish(200) + t.Run("ArchiveAfterFailover", func(t *testing.T) { + calls := export() + if len(calls) != 1 || calls[0].Get("status").Int() != 200 || calls[0].Get("failed").Bool() || calls[0].Get("response.id").String() != "healthy-response" || len(calls[0].Get("attempts").Array()) != 2 || calls[0].Get("attempts.0.status").Int() != int64(tc.status) || calls[0].Get("attempts.1.status").Int() != 200 { + t.Fatalf("failover archive: %v", calls) + } + }) if !completed { t.Fatal("healthy account never completed") } diff --git a/internal/runtime/executor/codex_websockets_duplex_test.go b/internal/runtime/executor/codex_websockets_duplex_test.go index 94fdb2c39d1..6ffa9f484a6 100644 --- a/internal/runtime/executor/codex_websockets_duplex_test.go +++ b/internal/runtime/executor/codex_websockets_duplex_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/MoeForge/moego/base64" "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/config" @@ -30,6 +31,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { steer := []byte(`{"type":"response.steer","previous_response_id":"r1","input":[{"role":"user","content":[{"type":"input_text","text":"STEER_OK"}]}]}`) accepted := []byte(`{"type":"response.steer.accepted","sequence_number":2,"steer":{"id":"s1","previous_response_id":"r1"}}`) pending := []byte(`{"type":"response.steer.pending","sequence_number":8,"steer":{"id":"s2","previous_response_id":"r2"},"reason":"waiting_for_required_input","required_input":[{"type":"function_call_output","call_id":"c1","name":"lookup"}]}`) + knownFailure := []byte(`{"type":"response.steer.failed","status":409,"steer":{"id":"s4","previous_response_id":"r3"},"error":{"code":"invalid_input"}}`) failed := []byte(`{"type":"response.steer.failed","sequence_number":14,"steer":{"id":"s3","previous_response_id":"missing","input":"recover me"},"error":{"code":"response_not_found","message":"missing response"}}`) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer close(serverDone) @@ -88,6 +90,10 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { t.Errorf("idle control missing: %s", got) } write(failed) + if got = read(); ojson.GetBytes(got, "previous_response_id").String() != "r3" { + t.Errorf("expected steer for r3: %s", got) + } + write(knownFailure) // Keep the socket alive until the client closes it; no automatic replay. _, _, _ = conn.ReadMessage() })) @@ -105,6 +111,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { exec.store = &codexWebsocketSessionStore{sessions: make(map[string]*codexWebsocketSession)} auth := &cliproxyauth.Auth{ID: "A", Provider: "codex", Attributes: map[string]string{"api_key": "test-key", "base_url": server.URL, "websockets": "true"}} req := core.Request{Model: "gpt-6-astra", Payload: []byte(`{"model":"gpt-6-astra","input":[],"instructions":"Initial settings"}`)} + ctx, initialCapture, export := beginTestDuplexArchive(t, ctx, req.Payload) opts := core.Options{SourceFormat: sdktranslator.FromString("codex"), Metadata: map[string]any{core.ExecutionSessionMetadataKey: "duplex-test"}} result, err := exec.ExecuteStream(ctx, auth, req, opts) if err != nil { @@ -146,11 +153,13 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { } seen["successor"] = true input <- core.WebsocketInput{Payload: []byte(`{"type":"response.steer","previous_response_id":"missing","input":"recover me"}`)} + case event == "response.steer.failed" && bytes.Equal(chunk.Payload, knownFailure): + seen["failed"] = true case event == "response.steer.failed": if !bytes.Equal(chunk.Payload, failed) { t.Fatal("failed input changed") } - seen["failed"] = true + input <- core.WebsocketInput{Payload: []byte(`{"type":"response.steer","previous_response_id":"r3","input":"reject this"}`)} } case <-ctx.Done(): t.Fatal(ctx.Err()) @@ -164,6 +173,33 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { } for range result.Chunks { } + initialCapture.Finish(200) + t.Run("ArchiveSteerOutcomes", func(t *testing.T) { + steers := map[string]ojson.Result{} + for _, call := range export() { + if call.Get("client_request.type").String() == "response.steer" { + steers[call.Get("client_request.previous_response_id").String()] = call + } + } + for parent, wantID := range map[string]string{"r1": "r2", "r2": "r3", "missing": "", "r3": ""} { + call := steers[parent] + if call.Get("upstream_request.previous_response_id").String() != parent { + t.Errorf("missing upstream steer for %s", parent) + } + if wantID == "" { + frame, err := base64.StdEncoding.DecodeString(call.Get("raw_frames_base64.0").String()) + wantStatus, wantFrame := int64(400), failed + if parent == "r3" { + wantStatus, wantFrame = 409, knownFailure + } + if !call.Get("failed").Bool() || call.Get("status").Int() != wantStatus || err != nil || !bytes.Equal(frame, wantFrame) { + t.Errorf("rejected steer = %s", call.Raw) + } + } else if call.Get("status").Int() != 200 || call.Get("response.id").String() != wantID { + t.Errorf("parent %s = %s", parent, call.Raw) + } + } + }) if connections.Load() != 1 { t.Fatalf("connections=%d", connections.Load()) } diff --git a/internal/runtime/executor/helps/logging_helpers.go b/internal/runtime/executor/helps/logging_helpers.go index 0ec205ff027..e5b5bb3d6ec 100644 --- a/internal/runtime/executor/helps/logging_helpers.go +++ b/internal/runtime/executor/helps/logging_helpers.go @@ -68,6 +68,10 @@ func requestLogCaptureEnabled(cfg *config.Config) bool { // RecordAPIRequest stores the upstream request metadata in Gin context for request logging. func RecordAPIRequest(ctx context.Context, cfg *config.Config, info UpstreamRequestLog) { tracearchive.From(ctx).Request(info.Provider, "http", info.Body) + recordAPIRequestLog(ctx, cfg, info) +} + +func recordAPIRequestLog(ctx context.Context, cfg *config.Config, info UpstreamRequestLog) { if cfg == nil || cfg.CommercialMode { return } @@ -405,7 +409,7 @@ func RecordAPIWebsocketRequest(ctx context.Context, cfg *config.Config, info Ups return } if !cfg.RequestLog { - RecordAPIRequest(ctx, cfg, info) + recordAPIRequestLog(ctx, cfg, info) return } ginCtx := ginContextFrom(ctx) diff --git a/internal/runtime/executor/helps/usage_helpers.go b/internal/runtime/executor/helps/usage_helpers.go index 5f7cd8e41c3..1d744933925 100644 --- a/internal/runtime/executor/helps/usage_helpers.go +++ b/internal/runtime/executor/helps/usage_helpers.go @@ -370,7 +370,8 @@ func (r *UsageReporter) PublishAdditionalModel(ctx context.Context, model string if !ok { return } - r.publishRecord(ctx, record) + // Supplemental billing is delivered to usage sinks independently of the primary attempt observer. + r.publishRecord(usage.WithRecordObserver(ctx, nil), record) } // SetTranslatedRequestMetadata captures usage-relevant values from the final diff --git a/internal/tracearchive/archive_test.go b/internal/tracearchive/archive_test.go index 04ba63aceb0..623d860b989 100644 --- a/internal/tracearchive/archive_test.go +++ b/internal/tracearchive/archive_test.go @@ -14,6 +14,7 @@ import ( "filippo.io/age" "github.com/MoeForge/moego/ojson" + "github.com/klauspost/compress/zstd" "github.com/rune-oai/t3code-proxy/internal/config" ) @@ -23,9 +24,18 @@ func check(t *testing.T, err error) { t.Fatal(err) } } +func archiveIdentity(t *testing.T) *age.X25519Identity { + t.Helper() + id, err := age.GenerateX25519Identity() + check(t, err) + return id +} func recorder(t *testing.T, settings config.TraceArchiveConfig) *Recorder { t.Helper() settings.Enabled = true + if len(settings.Recipients) == 0 { + settings.Recipients = []string{archiveIdentity(t).Recipient().String()} + } r := new(Recorder) check(t, r.Apply(&config.Config{TraceArchive: settings})) t.Cleanup(r.Close) @@ -134,22 +144,23 @@ func TestTornTailAndStaleSeal(t *testing.T) { _, err = s.f.WriteString(`{"type":"blo`) check(t, err) check(t, s.f.Close()) - r := recorder(t, config.TraceArchiveConfig{Dir: dir}) + id := archiveIdentity(t) + r := recorder(t, config.TraceArchiveConfig{Dir: dir, Recipients: []string{id.Recipient().String()}}) submit(t, r, `{"input":["old"]}`, `{"ok":true}`) r.Close() var b bytes.Buffer - check(t, command([]string{"verify", "--dir", dir}, &b, io.Discard)) + check(t, readArchive(dir, "", "", "verify", "calls", []age.Identity{id}, &b)) if strings.Count(b.String(), "1 calls, 1 blobs") != 2 { t.Fatal(b.String()) } - paths, err := filepath.Glob(filepath.Join(dir, "*.jsonl.zst")) + paths, err := filepath.Glob(filepath.Join(dir, "*.jsonl.zst.age")) check(t, err) f, err := os.OpenFile(paths[0], os.O_APPEND|os.O_WRONLY, 0600) check(t, err) _, err = f.WriteString("x") check(t, err) check(t, f.Close()) - if command([]string{"verify", "--dir", dir}, io.Discard, io.Discard) == nil { + if readArchive(dir, "", "", "verify", "calls", []age.Identity{id}, io.Discard) == nil { t.Fatal("corrupt sealed file accepted") } } @@ -247,7 +258,8 @@ func TestUploadRetryAndRetention(t *testing.T) { flag := filepath.Join(t.TempDir(), "fail") t.Setenv("FAIL_RCLONE", flag) check(t, os.WriteFile(flag, nil, 0600)) - w := &worker{cfg: config.TraceArchiveConfig{Dir: dir, MaxLocalSizeMB: 1, Upload: config.TraceUploadConfig{Enabled: true, Binary: fake, Remote: remote}}} + id := archiveIdentity(t) + w := &worker{cfg: config.TraceArchiveConfig{Dir: dir, Recipients: []string{id.Recipient().String()}, MaxLocalSizeMB: 1, Upload: config.TraceUploadConfig{Enabled: true, Binary: fake, Remote: remote}}} s, err := newSegment(dir, "2000-01-01") check(t, err) check(t, s.seal(w.cfg)) @@ -277,5 +289,131 @@ func TestUploadRetryAndRetention(t *testing.T) { if len(ms) != 1 || ms[0].UploadedTo != remote { t.Fatalf("unconfirmed remote manifest: %+v", ms) } - check(t, readArchive(remote, "", "", "verify", "calls", nil, io.Discard)) + check(t, readArchive(remote, "", "", "verify", "calls", []age.Identity{id}, io.Discard)) +} + +func TestMCPRequestCredentialsMaskedBeforePacking(t *testing.T) { + dir := t.TempDir() + id := archiveIdentity(t) + r := recorder(t, config.TraceArchiveConfig{Dir: dir, Recipients: []string{id.Recipient().String()}}) + for _, body := range []string{`{"mcp_servers":[{"type":"url","authorization_token":"claude-secret"}]}`, `{"tools":[{"type":"mcp","authorization":"responses-secret"}]}`} { + submit(t, r, body, `{"ok":true}`) + } + check(t, r.Seal(context.Background())) + ms, err := listManifests(dir) + check(t, err) + f, err := os.Open(filepath.Join(dir, ms[0].File)) + check(t, err) + defer closeLogged(f) + plain, err := age.Decrypt(f, id) + check(t, err) + z, err := zstd.NewReader(plain) + check(t, err) + defer z.Close() + raw, err := io.ReadAll(z) + check(t, err) + var out bytes.Buffer + check(t, readArchive(dir, "", "", "export", "calls", []age.Identity{id}, &out)) + for _, b := range [][]byte{raw, out.Bytes()} { + if bytes.Contains(b, []byte("-secret")) { + t.Fatal("credential present in archive") + } + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + for i, path := range []string{"mcp_servers.0.authorization_token", "tools.0.authorization"} { + for _, side := range []string{"client_request", "upstream_request"} { + if ojson.Get(lines[i], side+"."+path).String() != "[masked]" { + t.Fatalf("%s.%s not masked", side, path) + } + } + } +} + +func TestCodexMetadataFramesReconstruct(t *testing.T) { + frames := [][]byte{ + []byte(`{"type":"codex.response.metadata","headers":{"x-models-etag":"models-v1","x-codex-turn-state":"turn-1"}}`), + []byte(`{"type":"keepalive"}`), + []byte(`{"type":"response.metadata"}`), + []byte(`{"type":"response.completed","response":{"id":"r","output":[]}}`), + } + if got, ok := reconstruct(frames); !ok || ojson.GetBytes(got, "id").String() != "r" { + t.Fatalf("response = %s", got) + } +} + +func TestIncompleteItemShellPreservesDeltas(t *testing.T) { + for _, tc := range []struct{ event, shell string }{ + {"response.output_text.delta", `{"type":"message","content":[{"type":"output_text","text":""}]}`}, + {"response.function_call_arguments.delta", `{"type":"function_call","arguments":""}`}, + } { + frames := [][]byte{fmt.Appendf(nil, `{"type":%q,"output_index":0,"content_index":0,"delta":"partial"}`, tc.event), fmt.Appendf(nil, `{"type":"response.incomplete","response":{"output":[%s]}}`, tc.shell)} + if got, ok := reconstruct(frames); ok && !bytes.Contains(got, []byte("partial")) { + t.Fatalf("%s discarded delta: %s", tc.event, got) + } + } +} + +func TestArchiveOutsideAuthTreeAndNonJSONManifest(t *testing.T) { + cfg := &config.Config{} + cfg.AuthDir = filepath.Join(t.TempDir(), "auth") + dir, err := Directory(cfg) + check(t, err) + if filepath.Dir(dir) != filepath.Dir(cfg.AuthDir) || dir == cfg.AuthDir { + t.Errorf("expected sibling of %s, got %s", cfg.AuthDir, dir) + } + cfg.TraceArchive.Dir = filepath.Join(cfg.AuthDir, "custom") + dir, err = Directory(cfg) + check(t, err) + if dir != cfg.TraceArchive.Dir { + t.Fatal(dir) + } + r := recorder(t, config.TraceArchiveConfig{Dir: dir}) + submit(t, r, `{}`, `{}`) + check(t, r.Seal(context.Background())) + matches, err := filepath.Glob(filepath.Join(dir, "*.json")) + check(t, err) + if len(matches) > 0 { + t.Fatalf("auth store would read %v", matches) + } + r.Close() + ms, err := listManifests(dir) + check(t, err) + if len(ms) != 1 { + t.Fatalf("sealed manifests = %v", ms) + } + path := filepath.Join(dir, ms[0].File+manifestSuffix) + check(t, os.Rename(path, path+".json")) + check(t, readArchive(dir, "", "", "stats", "calls", nil, io.Discard)) + r = recorder(t, config.TraceArchiveConfig{Dir: dir}) + if len(r.Status().Segments) != 1 { + t.Fatal("lost legacy manifest") + } + if _, err = os.Stat(path + ".json"); !os.IsNotExist(err) { + t.Fatalf("legacy JSON sidecar remains: %v", err) + } +} + +func TestRecoverEvictionBeforeLiveReconciliation(t *testing.T) { + dir := t.TempDir() + cfg := config.TraceArchiveConfig{Dir: dir, Recipients: []string{archiveIdentity(t).Recipient().String()}} + s, err := newSegment(dir, today()) + check(t, err) + check(t, s.append(&call{Metadata: Metadata{SchemaVersion: 1}, client: []byte(`{}`)})) + live, err := os.ReadFile(s.path) + check(t, err) + check(t, s.seal(cfg)) + ms, err := listManifests(dir) + check(t, err) + path := filepath.Join(dir, ms[0].File) + check(t, os.Rename(path, path+".evicting")) + check(t, os.WriteFile(s.path, live, 0600)) + r := recorder(t, cfg) + if r.Status().Calls != 1 { + t.Fatal(r.Status()) + } + if _, err = os.Stat(s.path); !os.IsNotExist(err) { + t.Fatalf("live segment remains: %v", err) + } + _, err = os.Stat(path) + check(t, err) } diff --git a/internal/tracearchive/capture.go b/internal/tracearchive/capture.go index d20a2766f0f..3a2e748c0ad 100644 --- a/internal/tracearchive/capture.go +++ b/internal/tracearchive/capture.go @@ -84,11 +84,12 @@ func (r *Recorder) Bind(req *http.Request, protocol, traceID string) context.Con } type Capture struct { - mu sync.Mutex // protects only this call's snapshots, never storage or queue waits - run *worker - source source - finished bool - data *call + mu sync.Mutex // protects only this call's snapshots, never storage or queue waits + run *worker + source source + finished bool + duplexStatus int // business outcome of the handler-owned turn after bootstrap + data *call } const maxCaptureBytes = 128 << 20 @@ -185,6 +186,7 @@ func (c *Capture) Request(provider, transport string, p []byte) { defer c.mu.Unlock() if c.reserve(len(p)) { d := c.data + c.duplexStatus = 0 d.Provider = provider d.Attempts = append(d.Attempts, Attempt{Start: time.Now().UTC(), Provider: provider, Transport: transport}) d.upstream = bytes.Clone(p) @@ -279,6 +281,31 @@ func (c *Capture) observe(r usage.Record) { } d.Model = r.Model } + +// MarkDuplexResult lets the handler finish its initial turn once a committed +// response ends, even while the socket continues serving follow-up turns. +func (c *Capture) MarkDuplexResult(status int) { + if c == nil { + return + } + c.mu.Lock() + c.duplexStatus = status + c.mu.Unlock() +} + +// FinishDuplexResult commits a marked turn after the handler delivers its terminal frame. +func (c *Capture) FinishDuplexResult() { + if c == nil { + return + } + c.mu.Lock() + status := c.duplexStatus + c.mu.Unlock() + if status != 0 { + c.Finish(status) + } +} + func (c *Capture) Finish(status int) { if c == nil { return diff --git a/internal/tracearchive/json.go b/internal/tracearchive/json.go index 5af2518eadb..2e28518156f 100644 --- a/internal/tracearchive/json.go +++ b/internal/tracearchive/json.go @@ -119,7 +119,8 @@ func safePayload(data []byte) []byte { } name := strings.ToLower(strings.ReplaceAll(key, "_", "-")) lineHeader, _, hasColon := strings.Cut(v.Str, ":") - if headers && (credentialHeader(name) || named && (name == "value" || name == "values") || tuple && key != "0" || hasColon && credentialHeader(lineHeader)) { + structuredCredential := v.Type == ojson.String && (name == "authorization" || name == "authorization-token") + if structuredCredential || headers && (credentialHeader(name) || named && (name == "value" || name == "values") || tuple && key != "0" || hasColon && credentialHeader(lineHeader)) { _ = d.Set(p, "[masked]") } else { walk(v, p, headers || strings.Contains(name, "headers") || name == "client-metadata" || name == "metadata") diff --git a/internal/tracearchive/recorder.go b/internal/tracearchive/recorder.go index 7236d41dacf..809b768a8b3 100644 --- a/internal/tracearchive/recorder.go +++ b/internal/tracearchive/recorder.go @@ -55,7 +55,14 @@ func Directory(c *config.Config) (string, error) { if err != nil { return "", err } - return filepath.Join(dir, "traces"), nil + dir, err = filepath.Abs(dir) + if err != nil { + return "", err + } + if filepath.Dir(dir) == dir { + return "", fmt.Errorf("trace-archive: set dir explicitly when auth-dir is a filesystem root") + } + return dir + "-traces", nil } func (r *Recorder) Apply(c *config.Config) error { r.mu.Lock() @@ -116,6 +123,24 @@ func (w *worker) recover() error { if err != nil { return err } + // Upgrade sidecars in place while holding the writer lock. Offline readers + // also accept the old suffix so historical exports need no migration. + for _, e := range entries { + name, ok := strings.CutSuffix(e.Name(), manifestSuffix+".json") + if !ok || !segmentName.MatchString(name) { + continue + } + path := filepath.Join(w.cfg.Dir, name+manifestSuffix) + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot migrate trace manifest %s", e.Name()) + } + if err := os.Rename(path+".json", path); err != nil { + return err + } + } + if err := recoverEvictions(w.cfg.Dir); err != nil { + return err + } ms, err := listManifests(w.cfg.Dir) if err != nil { return err @@ -176,11 +201,7 @@ func (w *worker) recover() error { } // A crash before manifest publication can leave an orphan with the old // encryption suffix. The replacement is durable before removing the orphan. - orphan := target + ".age" - if len(w.cfg.Recipients) > 0 { - orphan = target - } - if err = os.Remove(orphan); err != nil && !errors.Is(err, os.ErrNotExist) { + if err = os.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } diff --git a/internal/tracearchive/response.go b/internal/tracearchive/response.go index ee9a691dda9..ed31f542aa3 100644 --- a/internal/tracearchive/response.go +++ b/internal/tracearchive/response.go @@ -19,7 +19,7 @@ func reconstruct(chunks [][]byte) ([]byte, bool) { if len(chunks) == 1 && ojson.ValidBytes(chunks[0]) { r := ojson.ParseBytes(chunks[0]) typ := r.Get("type").String() - if !strings.HasPrefix(typ, "response.") && !strings.HasPrefix(typ, "content_block_") && !strings.HasPrefix(typ, "message_") && !r.Get("choices.0.delta").Exists() && typ != "ping" { + if !strings.HasPrefix(typ, "response.") && !strings.HasPrefix(typ, "content_block_") && !strings.HasPrefix(typ, "message_") && !r.Get("choices.0.delta").Exists() && typ != "ping" && typ != "keepalive" && typ != "codex.response.metadata" && typ != "codex.rate_limits" { return safePayload(chunks[0]), true } } @@ -58,6 +58,11 @@ func reconstruct(chunks [][]byte) ([]byte, bool) { return nil, false } } + for path, delta := range s.deltas { + if final := s.doc.Get(path); final.Type != ojson.String || final.String() != delta.String() { + return nil, false + } + } return s.doc.Bytes(), true } @@ -67,6 +72,7 @@ type streamState struct { complete, chat, chatEnd bool items map[int]string observedItems map[int]bool + deltas map[string]*strings.Builder openBlocks map[int]*strings.Builder } @@ -84,7 +90,7 @@ func (s *streamState) event(r ojson.Result) bool { s.observedItems[int(n)] = true } switch { - case typ == "ping": + case typ == "ping", typ == "keepalive", typ == "codex.response.metadata", typ == "response.metadata": return true case typ == "message_start": msg := r.Get("message") @@ -141,6 +147,36 @@ func (s *streamState) event(r ojson.Result) bool { mergeFields(s.doc, "usage.", r.Get("usage")) case typ == "message_stop": s.complete = true + case strings.HasPrefix(typ, "response.") && strings.HasSuffix(typ, ".delta"): + // Drop raw frames only when the indexed final field includes the observed deltas. + path := "output." + strconv.FormatInt(idx.Int(), 10) + "." + switch typ { + case "response.output_text.delta": + path += "content." + strconv.FormatInt(r.Get("content_index").Int(), 10) + ".text" + case "response.function_call_arguments.delta", "response.mcp_call_arguments.delta": + path += "arguments" + case "response.custom_tool_call_input.delta": + path += "input" + case "response.refusal.delta": + path += "content." + strconv.FormatInt(r.Get("content_index").Int(), 10) + ".refusal" + case "response.reasoning_text.delta": + path += "content." + strconv.FormatInt(r.Get("content_index").Int(), 10) + ".text" + case "response.reasoning_summary_text.delta": + path += "summary." + strconv.FormatInt(r.Get("summary_index").Int(), 10) + ".text" + default: + return false + } + delta := r.Get("delta") + if delta.Type != ojson.String { + return false + } + if s.deltas == nil { + s.deltas = make(map[string]*strings.Builder) + } + if s.deltas[path] == nil { + s.deltas[path] = new(strings.Builder) + } + s.deltas[path].WriteString(delta.String()) case typ == "response.output_item.done": item := r.Get("item") if !item.IsObject() { diff --git a/internal/tracearchive/segment.go b/internal/tracearchive/segment.go index b78d9a640ff..666238ecf63 100644 --- a/internal/tracearchive/segment.go +++ b/internal/tracearchive/segment.go @@ -21,6 +21,8 @@ import ( log "github.com/sirupsen/logrus" ) +const manifestSuffix = ".manifest" + var segmentName = regexp.MustCompile(`^traces-(\d{4}-\d{2}-\d{2})(-\d+)?\.(live|jsonl\.zst(\.age)?)$`) type counts struct { @@ -315,14 +317,14 @@ func syncDirectory(dir string) error { return f.Sync() } func writeManifest(dir string, m Manifest) error { - return atomicFile(filepath.Join(dir, m.File+".manifest.json"), func(f *os.File) error { _, err := f.Write(m.bytes()); return err }) + return atomicFile(filepath.Join(dir, m.File+manifestSuffix), func(f *os.File) error { _, err := f.Write(m.bytes()); return err }) } func (s *segment) seal(cfg config.TraceArchiveConfig) error { if err := s.f.Sync(); err != nil { return err } - target := strings.TrimSuffix(s.path, ".live") + ".jsonl.zst" + target := strings.TrimSuffix(s.path, ".live") + ".jsonl.zst.age" var keys []age.Recipient for _, key := range cfg.Recipients { k, err := age.ParseX25519Recipient(key) @@ -331,26 +333,15 @@ func (s *segment) seal(cfg config.TraceArchiveConfig) error { } keys = append(keys, k) } - if len(keys) > 0 { - target += ".age" - } else { - log.Warn("sealing plaintext trace archive: no age recipients configured") - } footer := entry("footer") _ = footer.Set("counts", s.counts) _ = footer.Set("sha256", s.checksum()) if err := atomicFile(target, func(f *os.File) error { - var out io.Writer = f - var enc io.WriteCloser - if len(keys) > 0 { - var err error - enc, err = age.Encrypt(f, keys...) - if err != nil { - return err - } - out = enc + enc, err := age.Encrypt(f, keys...) + if err != nil { + return err } - z, err := zstd.NewWriter(out, zstd.WithEncoderConcurrency(1)) + z, err := zstd.NewWriter(enc, zstd.WithEncoderConcurrency(1)) if err != nil { return err } @@ -363,10 +354,8 @@ func (s *segment) seal(cfg config.TraceArchiveConfig) error { if e := z.Close(); err == nil { err = e } - if enc != nil { - if e := enc.Close(); err == nil { - err = e - } + if e := enc.Close(); err == nil { + err = e } return err }); err != nil { @@ -394,7 +383,10 @@ func listManifests(dir string) ([]Manifest, error) { } var out []Manifest for _, e := range entries { - file, ok := strings.CutSuffix(e.Name(), ".manifest.json") + file, ok := strings.CutSuffix(e.Name(), manifestSuffix) + if !ok { + file, ok = strings.CutSuffix(e.Name(), manifestSuffix+".json") + } if !ok { continue } diff --git a/internal/tracearchive/upload.go b/internal/tracearchive/upload.go index b19fec52f43..f62fc8014da 100644 --- a/internal/tracearchive/upload.go +++ b/internal/tracearchive/upload.go @@ -62,7 +62,7 @@ func (w *worker) maintenance(ctx context.Context) error { if ctx.Err() != nil { return ctx.Err() } - if w.cfg.Upload.Enabled && m.UploadedTo != w.cfg.Upload.Remote { + if w.cfg.Upload.Enabled && m.UploadedTo != w.cfg.Upload.Remote && strings.HasSuffix(m.File, ".age") { if err = w.upload(ctx, &m); err != nil { first = err break @@ -121,7 +121,7 @@ func (w *worker) upload(ctx context.Context, m *Manifest) error { if err != nil { return err } - if err = copyVerified(tmp.Name(), m.File+".manifest.json", digest(raw), int64(len(raw))); err != nil { + if err = copyVerified(tmp.Name(), m.File+manifestSuffix, digest(raw), int64(len(raw))); err != nil { return err } if err = writeManifest(w.cfg.Dir, confirmed); err == nil { @@ -156,7 +156,7 @@ func recoverEvictions(dir string) error { continue } path := filepath.Join(dir, name) - _, err = os.Stat(path + ".manifest.json") + _, err = os.Stat(path + manifestSuffix) switch { case err == nil: if _, exists := os.Lstat(path); !errors.Is(exists, os.ErrNotExist) { @@ -184,7 +184,7 @@ func (w *worker) retain(ms []Manifest) error { } var total int64 for _, e := range entries { - if segmentName.MatchString(strings.TrimSuffix(e.Name(), ".manifest.json")) { + if segmentName.MatchString(strings.TrimSuffix(e.Name(), manifestSuffix)) { i, err := e.Info() if err != nil { return err @@ -200,7 +200,7 @@ func (w *worker) retain(ms []Manifest) error { continue } path := filepath.Join(w.cfg.Dir, m.File) - manifest := path + ".manifest.json" + manifest := path + manifestSuffix sidecar, err := os.Stat(manifest) if err != nil { return err diff --git a/sdk/api/handlers/openai/openai_responses_websocket.go b/sdk/api/handlers/openai/openai_responses_websocket.go index 286eafc6b0e..ae3e7b88ed6 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket.go +++ b/sdk/api/handlers/openai/openai_responses_websocket.go @@ -1108,6 +1108,7 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { wsTimelineLog, passthroughSessionID, responsesWebsocketForwardOptions{ + initialCapture: archiveTurn, preserveCompletionOutput: preserveNativeOutput.Load, duplexStream: func() bool { return codexDuplexStream.Load() && !cliproxyexecutor.LocalWebsocketTurnExecuted(cliCtx) @@ -1122,12 +1123,12 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { }, ) cliCancel(nil) - if errForward != nil { + if forwardErrMsg != nil && (!failoverInPlace || cliproxyexecutor.IsUpstreamWebsocketTurnError(forwardErrMsg.Error)) { + archiveTurn.Finish(forwardErrMsg.StatusCode) + } else if errForward != nil { archiveTurn.Finish(http.StatusBadGateway) } else if forwardErrMsg == nil { archiveTurn.Finish(http.StatusOK) - } else if !failoverInPlace || cliproxyexecutor.IsUpstreamWebsocketTurnError(forwardErrMsg.Error) { - archiveTurn.Finish(forwardErrMsg.StatusCode) } if errForward != nil { wsTerminateErr = errForward diff --git a/sdk/api/handlers/openai/openai_responses_websocket_forward.go b/sdk/api/handlers/openai/openai_responses_websocket_forward.go index 49007b846f5..791dc63ed7d 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket_forward.go +++ b/sdk/api/handlers/openai/openai_responses_websocket_forward.go @@ -16,6 +16,7 @@ import ( "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/clienterror" "github.com/rune-oai/t3code-proxy/internal/interfaces" + "github.com/rune-oai/t3code-proxy/internal/tracearchive" "github.com/rune-oai/t3code-proxy/internal/util" "github.com/rune-oai/t3code-proxy/sdk/api/handlers" coreauth "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" @@ -24,6 +25,7 @@ import ( ) type responsesWebsocketForwardOptions struct { + initialCapture *tracearchive.Capture preserveCompletionOutput func() bool duplexStream func() bool toolCacheTurn *responsesWebsocketToolCacheTurn @@ -352,6 +354,9 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( cancel(errWrite) return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), nil, errWrite } + if turnEnded { + opts.initialCapture.FinishDuplexResult() + } } } } diff --git a/sdk/cliproxy/usage/manager.go b/sdk/cliproxy/usage/manager.go index 839f552e781..4cfd4c9f594 100644 --- a/sdk/cliproxy/usage/manager.go +++ b/sdk/cliproxy/usage/manager.go @@ -726,6 +726,9 @@ func (m *Manager) RegisterNamed(name string, plugin Plugin) { // WithRecordObserver attaches a request-local observer for metadata that must // be joined before an HTTP handler or websocket turn completes. func WithRecordObserver(ctx context.Context, observe func(Record)) context.Context { + if ctx == nil { + ctx = context.Background() + } return context.WithValue(ctx, recordObserverKey{}, observe) } diff --git a/web/src/i18n/locales/en.json b/web/src/i18n/locales/en.json index cd6f0662a72..76934a923a5 100644 --- a/web/src/i18n/locales/en.json +++ b/web/src/i18n/locales/en.json @@ -1414,9 +1414,9 @@ "request_log": "Request logs", "trace_archive": "Trace archive", "trace_archive_record": "Archive model calls", - "trace_archive_desc": "Save prompts, tool calls and responses. Set age recipients in YAML to encrypt sealed archives.", + "trace_archive_desc": "Save prompts, tool calls and responses. Configure at least one age X25519 recipient in YAML before enabling the archive.", "trace_archive_dir": "Archive directory", - "trace_archive_dir_hint": "Defaults to the traces folder in your auth directory.", + "trace_archive_dir_hint": "Defaults to a sibling of your auth directory with -traces appended to its name.", "trace_archive_status": "{{calls}} calls today · {{queued}} queued · {{dropped}} dropped", "trace_archive_status_failed": "Archive status unavailable", "request_log_desc": "Record full request and response bodies. Use only while debugging; logs can contain prompts.", From ef3df971ad64c1fb6df3fa897823b2dcb50fc922 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 20 Nov 2026 09:04:39 +0900 Subject: [PATCH 3/3] Fix duplex trace ownership for queued rejections and steered responses --- .../executor/codex_websockets_duplex.go | 91 +++++++++---------- ...bsockets_duplex_credential_failure_test.go | 63 +++++++++++++ .../executor/codex_websockets_duplex_test.go | 38 +++++++- internal/tracearchive/capture.go | 37 ++++++-- 4 files changed, 172 insertions(+), 57 deletions(-) diff --git a/internal/runtime/executor/codex_websockets_duplex.go b/internal/runtime/executor/codex_websockets_duplex.go index 9c4c85b71ce..59c64c15d48 100644 --- a/internal/runtime/executor/codex_websockets_duplex.go +++ b/internal/runtime/executor/codex_websockets_duplex.go @@ -820,11 +820,13 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( // Bootstrap rejections can still retry. The handler owns the initial // capture across those attempts and throughout the committed turn. finishResponse := func(owner *codexWebsocketPrepared, status int) { + if owner == initial && !firstResponse { + owner.archive.MarkDuplexResult(status) + return + } owner.archive.Status(status) if owner != initial { owner.archive.Finish(status) - } else if !firstResponse { - owner.archive.MarkDuplexResult(status) } } defer close(out) @@ -859,27 +861,24 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( return false } } - logFrame := func(capture *tracearchive.Capture, payload []byte) context.Context { - frameCtx := capture.Context(ctx) + logFrame := func(frameCtx context.Context, payload []byte) context.Context { helps.AppendCodexAPIWebsocketResponse(frameCtx, e.cfg, payload) helps.EmitWebSocketResponseEvent(ctx, opts, auth, e.Identifier(), req.Model, payload) return frameCtx } var successorCaptures []*tracearchive.Capture + responseCtx := ctx recordFrame := func(owner *codexWebsocketPrepared, payload []byte) context.Context { - frameCapture := owner.archive - if owner == current { - // Several steers can be consumed by the same successor, including - // one started by an explicit create with tool results. - for _, capture := range successorCaptures { - if frameCapture == nil { - frameCapture = capture - } else { - capture.Chunk(payload) - } + if owner != current { + return logFrame(owner.archive.Context(ctx), payload) + } + primary := tracearchive.From(responseCtx) + for _, capture := range successorCaptures { + if capture != primary { + capture.Chunk(payload) } } - return logFrame(frameCapture, payload) + return logFrame(responseCtx, payload) } finishSuccessorSteers := func(status int) { for _, capture := range successorCaptures { @@ -969,7 +968,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( connectionErr = turnErr } } - reporter.PublishFailure(current.archive.Context(ctx), connectionErr) + reporter.PublishFailure(responseCtx, connectionErr) send(cliproxyexecutor.StreamChunk{Err: connectionErr}) } return @@ -996,7 +995,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( if settings == nil { metadataMu.Unlock() connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("automatic successor has no retained parent settings")} - reporter.PublishFailure(current.archive.Context(ctx), connectionErr) + reporter.PublishFailure(responseCtx, connectionErr) send(cliproxyexecutor.StreamChunk{Err: connectionErr}) return } @@ -1018,6 +1017,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( if len(pending) > 0 { current, pending = pending[0], pending[1:] } + responseCtx = current.archive.SuccessorContext(ctx, successorCaptures) responseID = createdResponseID latestTurnResponseID = createdResponseID // Retain response settings, not request history or authorization headers. @@ -1089,7 +1089,7 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( waitingParent = parent } metadataMu.Unlock() - logFrame(submission.archive, payload) + logFrame(submission.archive.Context(ctx), payload) if eventType == "response.steer.failed" { status := int(ojson.GetBytesNoCopy(payload, "status").Int()) if status == 0 { @@ -1110,28 +1110,6 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( metadataMu.Lock() isInitialResponse := firstResponse metadataMu.Unlock() - if !isInitialResponse && (eventType == "error" || eventType == "response.failed") { - var credentialErr error - if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok { - credentialErr = wsErr - } else if streamErr, _, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok { - credentialErr = streamErr - } - var status interface{ StatusCode() int } - if errors.As(credentialErr, &status) && (status.StatusCode() == http.StatusUnauthorized || status.StatusCode() == http.StatusForbidden || status.StatusCode() == http.StatusTooManyRequests || isCodexOverloadBootstrapFailure(payload)) { - // Account health is independent of which queued request failed. - // The conductor records the original classification without replaying - // this already-started stream on another credential. - frameCtx := recordFrame(current, payload) - reporter.PublishFailure(frameCtx, credentialErr) - finishResponse(current, status.StatusCode()) - finishSuccessorSteers(status.StatusCode()) - if send(cliproxyexecutor.StreamChunk{Payload: payload}) { - send(cliproxyexecutor.StreamChunk{Err: credentialErr}) - } - return - } - } eventPrepared, eventReporter := current, reporter if !isInitialResponse && (eventType == "response.failed" || eventType == "error") { failedID := ojson.GetBytesNoCopy(payload, "response.id").String() @@ -1155,19 +1133,36 @@ func (e *CodexWebsocketsExecutor) streamCodexDuplex( terminalDeliveryPending = true } metadataMu.Unlock() - wakeWriter() - if ambiguous { - logFrame(nil, payload) - // Without a response ID, assigning this failure could corrupt - // either request. Preserve the event and fail the socket without - // guessing a scope, replaying input, or cooling the credential. - connectionErr := &codexDuplexConnectionError{cause: fmt.Errorf("cannot associate websocket failure with a response or pending create")} - reporter.PublishFailure(current.archive.Context(ctx), connectionErr) + var credentialErr error + if wsErr, ok := parseCodexWebsocketErrorWithCooling(payload, e.modelLevelCooling()); ok { + credentialErr = wsErr + } else if streamErr, _, ok := codexTerminalFailureErrWithCooling(payload, e.modelLevelCooling()); ok { + credentialErr = streamErr + } + var status interface{ StatusCode() int } + credentialFailure := errors.As(credentialErr, &status) && (status.StatusCode() == http.StatusUnauthorized || status.StatusCode() == http.StatusForbidden || status.StatusCode() == http.StatusTooManyRequests || isCodexOverloadBootstrapFailure(payload)) + if credentialFailure || ambiguous { + connectionErr := credentialErr + if ambiguous { + // Preserve account health classification even when no turn owns the frame. + if !credentialFailure { + connectionErr = &codexDuplexConnectionError{cause: fmt.Errorf("cannot associate websocket failure with a response or pending create")} + } + var unowned *tracearchive.Capture + eventReporter.PublishFailure(logFrame(unowned.Context(ctx), payload), connectionErr) + } else { + eventReporter.PublishFailure(recordFrame(eventPrepared, payload), connectionErr) + finishResponse(eventPrepared, status.StatusCode()) + if eventPrepared == current { + finishSuccessorSteers(status.StatusCode()) + } + } if send(cliproxyexecutor.StreamChunk{Payload: payload}) { send(cliproxyexecutor.StreamChunk{Err: connectionErr}) } return } + wakeWriter() } frameCtx := recordFrame(eventPrepared, payload) restoreMultiAgent := !eventPrepared.multiAgentV2Conflict && (eventPrepared.optimizeMultiAgentV2 || sess.isMultiAgentV2Optimized(conn)) diff --git a/internal/runtime/executor/codex_websockets_duplex_credential_failure_test.go b/internal/runtime/executor/codex_websockets_duplex_credential_failure_test.go index 2a4d273ca7e..49ec4eb564a 100644 --- a/internal/runtime/executor/codex_websockets_duplex_credential_failure_test.go +++ b/internal/runtime/executor/codex_websockets_duplex_credential_failure_test.go @@ -19,6 +19,69 @@ import ( translator "github.com/rune-oai/t3code-proxy/sdk/translator" ) +func TestCodexDuplexQueuedCredentialRejectionArchive(t *testing.T) { + const rejection = `{"type":"error","status":401,"error":{"type":"authentication_error","message":"queued create rejected"}}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer func() { _ = c.Close() }() + _ = c.SetReadDeadline(time.Now().Add(5 * time.Second)) + for _, frames := range [][]string{ + {`{"type":"response.created","response":{"id":"first","output":[]}}`, `{"type":"response.completed","response":{"id":"first","output":[]}}`}, + {rejection}, + } { + if _, _, err = c.ReadMessage(); err != nil { + t.Error(err) + return + } + for _, frame := range frames { + if err = c.WriteMessage(websocket.TextMessage, []byte(frame)); err != nil { + t.Error(err) + return + } + } + } + _, _, _ = c.ReadMessage() + })) + defer upstream.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + input := make(chan core.WebsocketInput, 1) + ctx = core.WithWebsocketInput(core.WithDownstreamWebsocket(ctx), input) + req := core.Request{Model: "gpt-6-astra", Payload: []byte(`{"model":"gpt-6-astra","input":[]}`)} + ctx, initial, export := beginTestDuplexArchive(t, ctx, req.Payload) + exec := NewCodexWebsocketsExecutor(&config.Config{Codex: config.CodexConfig{ResponseSteering: true}}) + credential := &auth.Auth{ID: t.Name(), Provider: "codex", Attributes: map[string]string{"api_key": "test", "base_url": upstream.URL, "websockets": "true"}} + result, err := exec.ExecuteStream(ctx, credential, req, core.Options{SourceFormat: translator.FromString("codex"), Metadata: map[string]any{core.ExecutionSessionMetadataKey: t.Name()}}) + if err != nil { + t.Fatal(err) + } + for chunk := range result.Chunks { + if ojson.GetBytes(chunk.Payload, "type").String() == "response.completed" { + input <- core.WebsocketInput{Payload: []byte(`{"type":"response.create","event_id":"queued","input":[]}`)} + } + } + // The handler can still be forwarding the first completion when the socket fails. + initial.Status(502) + initial.Finish(502) + calls := export() + if len(calls) != 2 { + t.Fatalf("got %d calls, want 2", len(calls)) + } + for _, call := range calls { + if call.Get("client_request.event_id").String() == "queued" { + if call.Get("status").Int() != 401 || !call.Get("failed").Bool() || call.Get("response.error.message").String() != "queued create rejected" { + t.Errorf("queued rejection = %s", call.Raw) + } + } else if call.Get("status").Int() != 200 || call.Get("attempts.0.status").Int() != 200 || call.Get("failed").Bool() || call.Get("response.id").String() != "first" { + t.Errorf("initial completion = %s", call.Raw) + } + } +} + func TestCodexDuplexLaterCredentialFailure(t *testing.T) { for _, kind := range []string{"error", "response.failed"} { for _, status := range []int{401, 403, 429} { diff --git a/internal/runtime/executor/codex_websockets_duplex_test.go b/internal/runtime/executor/codex_websockets_duplex_test.go index 6ffa9f484a6..a983d94b371 100644 --- a/internal/runtime/executor/codex_websockets_duplex_test.go +++ b/internal/runtime/executor/codex_websockets_duplex_test.go @@ -14,6 +14,7 @@ import ( "github.com/MoeForge/moego/ojson" "github.com/gorilla/websocket" "github.com/rune-oai/t3code-proxy/internal/config" + "github.com/rune-oai/t3code-proxy/internal/logging" cliproxyauth "github.com/rune-oai/t3code-proxy/sdk/cliproxy/auth" core "github.com/rune-oai/t3code-proxy/sdk/cliproxy/executor" coreusage "github.com/rune-oai/t3code-proxy/sdk/cliproxy/usage" @@ -65,6 +66,10 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { t.Errorf("steer transformed: %s", got) } write(accepted) + if got := read(); !bytes.Equal(got, steer) { + t.Errorf("second steer transformed: %s", got) + } + write([]byte(`{"type":"response.steer.accepted","steer":{"id":"s1b","previous_response_id":"r1"}}`)) write([]byte(fmt.Sprintf(`{"type":%q,"response":{"id":"r1","output":[],"incomplete_details":{"reason":"steered"}}}`, boundary))) write([]byte(`{"type":"response.created","response":{"id":"r2","output":[]}}`)) got := read() @@ -72,7 +77,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { t.Errorf("second control: %s", got) } write([]byte(`{"type":"response.steer.accepted","steer":{"id":"s2","previous_response_id":"r2"}}`)) - write([]byte(`{"type":"response.completed","response":{"id":"r2","output":[{"type":"function_call","call_id":"c1","name":"lookup","arguments":"{}"}]}}`)) + write([]byte(`{"type":"response.completed","response":{"id":"r2","usage":{"input_tokens":7,"output_tokens":2,"total_tokens":9},"output":[{"type":"function_call","call_id":"c1","name":"lookup","arguments":"{}"}]}}`)) write(pending) got = read() if ojson.GetBytes(got, "type").String() != "response.create" || ojson.GetBytes(got, "previous_response_id").String() != "r2" || ojson.GetBytes(got, "input.0.call_id").String() != "c1" { @@ -83,7 +88,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { } write([]byte(`{"type":"response.created","response":{"id":"r3","output":[]}}`)) write([]byte(`{"type":"response.output_item.done","output_index":0,"item":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"STEER_OK"}]}}`)) - write([]byte(`{"type":"response.completed","response":{"id":"r3","output":[]}}`)) + write([]byte(`{"type":"response.completed","response":{"id":"r3","usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12},"output":[]}}`)) // Idle steering must also reach the same live upstream connection. got = read() if ojson.GetBytes(got, "previous_response_id").String() != "missing" { @@ -103,6 +108,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { input := make(chan core.WebsocketInput, 8) ctx = core.WithWebsocketInput(core.WithDownstreamWebsocket(ctx), input) ctx = coreusage.WithRequestedModelAlias(ctx, t.Name()) + ctx = logging.WithClientRequestMetadata(ctx, logging.ClientRequestMetadata{SessionID: "steering-session"}) ctx = coreusage.WithHomeSelection(ctx, coreusage.HomeSelection{Model: "home-route"}) capture := &codexResponseModelUsageCapture{alias: t.Name(), records: make(chan coreusage.Record, 8)} coreusage.RegisterNamedPlugin(t.Name(), capture) @@ -132,6 +138,7 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { switch { case event == "response.created" && id == "r1": input <- core.WebsocketInput{Payload: steer} + input <- core.WebsocketInput{Payload: steer} case event == "response.steer.accepted" && ojson.GetBytes(chunk.Payload, "steer.id").String() == "s1": if !bytes.Equal(chunk.Payload, accepted) { t.Fatal("accepted event changed") @@ -174,9 +181,10 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { for range result.Chunks { } initialCapture.Finish(200) + calls := export() t.Run("ArchiveSteerOutcomes", func(t *testing.T) { steers := map[string]ojson.Result{} - for _, call := range export() { + for _, call := range calls { if call.Get("client_request.type").String() == "response.steer" { steers[call.Get("client_request.previous_response_id").String()] = call } @@ -204,8 +212,10 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { t.Fatalf("connections=%d", connections.Load()) } ids := make(map[string]bool) + records := make(map[int64]coreusage.Record) for turn := range 3 { record := capture.await(t) + records[record.Detail.TotalTokens] = record if record.HomeSelection == nil || record.HomeSelection.Reused != (turn > 0) || record.HomeSelection.Model != "home-route" { t.Fatalf("turn %d accounting=%+v", turn, record.HomeSelection) } @@ -214,6 +224,28 @@ func TestCodexDuplexSteeringLifecycle(t *testing.T) { } ids[record.RequestID] = true } + t.Run("ArchiveSuccessorUsage", func(t *testing.T) { + // A marker in the same sink bounds the count after all turns have finished. + coreusage.PublishRecord(ctx, coreusage.Record{Alias: capture.alias, RequestID: "end-of-test"}) + if next := capture.await(t); next.RequestID != "end-of-test" { + t.Fatalf("extra usage record for %s", next.RequestID) + } + counts := map[string]int{} + for _, call := range calls { + id := call.Get("response.id").String() + tokens, ok := map[string]int64{"r2": 9, "r3": 12}[id] + if !ok { + continue + } + counts[id]++ + if call.Get("token_usage.TotalTokens").Int() != tokens || call.Get("auth_index").String() != auth.EnsureIndex() || call.Get("session_id").String() != "steering-session" || call.Get("request_id").String() != records[tokens].RequestID { + t.Errorf("%s capture missing successor usage: %s", id, call.Raw) + } + } + if counts["r2"] != 2 || counts["r3"] != 2 { + t.Errorf("successor captures = %v, want two per response", counts) + } + }) for _, key := range []string{"accepted", "boundary", "pending", "successor", "failed"} { if !seen[key] { t.Fatalf("missing %s", key) diff --git a/internal/tracearchive/capture.go b/internal/tracearchive/capture.go index 3a2e748c0ad..9f442d11d5a 100644 --- a/internal/tracearchive/capture.go +++ b/internal/tracearchive/capture.go @@ -131,6 +131,24 @@ func (c *Capture) Context(ctx context.Context) context.Context { } return c.attach(ctx) } + +// SuccessorContext observes one usage record for the create and every steer it consumes. +func (c *Capture) SuccessorContext(ctx context.Context, steers []*Capture) context.Context { + if c == nil && len(steers) > 0 { + c, steers = steers[0], steers[1:] + } + ctx = c.Context(ctx) + if len(steers) == 0 { + return ctx + } + return usage.WithRecordObserver(ctx, func(record usage.Record) { + c.observe(record) + for _, steer := range steers { + steer.observe(record) + } + }) +} + func (c *Capture) attach(ctx context.Context) context.Context { ctx = context.WithValue(ctx, sourceKey{}, c.source) return usage.WithRecordObserver(context.WithValue(ctx, captureKey{}, c), c.observe) @@ -184,9 +202,8 @@ func (c *Capture) Request(provider, transport string, p []byte) { } c.mu.Lock() defer c.mu.Unlock() - if c.reserve(len(p)) { + if c.duplexStatus == 0 && c.reserve(len(p)) { d := c.data - c.duplexStatus = 0 d.Provider = provider d.Attempts = append(d.Attempts, Attempt{Start: time.Now().UTC(), Provider: provider, Transport: transport}) d.upstream = bytes.Clone(p) @@ -199,7 +216,7 @@ func (c *Capture) Status(status int) { } c.mu.Lock() defer c.mu.Unlock() - if d := c.data; d != nil && len(d.Attempts) > 0 { + if d := c.data; c.duplexStatus == 0 && d != nil && len(d.Attempts) > 0 { d.Attempts[len(d.Attempts)-1].Status = status } } @@ -228,7 +245,7 @@ func (c *Capture) chunk(p []byte, upstream bool) { c.mu.Lock() defer c.mu.Unlock() - if c.reserve(len(p) + 24) { + if (!upstream || c.duplexStatus == 0) && c.reserve(len(p)+24) { d := c.data if len(p) > 0 { d.markFirstByte() @@ -246,7 +263,7 @@ func (c *Capture) observe(r usage.Record) { c.mu.Lock() defer c.mu.Unlock() d := c.data - if d == nil { + if d == nil || c.duplexStatus != 0 { return } d.RequestID = r.RequestID @@ -289,7 +306,12 @@ func (c *Capture) MarkDuplexResult(status int) { return } c.mu.Lock() - c.duplexStatus = status + if c.duplexStatus == 0 { + c.duplexStatus = status + if d := c.data; d != nil && len(d.Attempts) > 0 { + d.Attempts[len(d.Attempts)-1].Status = status + } + } c.mu.Unlock() } @@ -316,6 +338,9 @@ func (c *Capture) Finish(status int) { return } c.finished = true + if c.duplexStatus != 0 { + status = c.duplexStatus + } d := c.data c.data = nil c.mu.Unlock()