Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"strings"
"time"

"github.com/rune-oai/t3code-proxy/internal/tracearchive"

"github.com/joho/godotenv"
configaccess "github.com/rune-oai/t3code-proxy/internal/access/config_access"
"github.com/rune-oai/t3code-proxy/internal/api"
Expand Down Expand Up @@ -89,6 +91,9 @@ func run() (exitStatus int) {

// "doctor" checks whether this machine is ready to act as a hub. It needs
// no running server and never reads credentials.
if len(os.Args) > 1 && os.Args[1] == "traces" {
return tracearchive.Main(os.Args[2:], os.Stdout, os.Stderr)
}
if len(os.Args) > 1 && os.Args[1] == "doctor" {
return doctor.Main(context.Background(), os.Args[2:], os.Stdout, os.Stderr)
}
Expand Down Expand Up @@ -190,7 +195,7 @@ func run() (exitStatus int) {
flag.CommandLine.Usage = func() {
out := flag.CommandLine.Output()
_, _ = fmt.Fprintf(out, "Usage of %s\n", os.Args[0])
_, _ = fmt.Fprint(out, "\nCommands:\n doctor Check whether this machine is ready to run as a hub (doctor -h for flags)\n discover Find gateways on the local network (discover -h for flags)\n\nFlags:\n")
_, _ = fmt.Fprint(out, "\nCommands:\n doctor Check whether this machine is ready to run as a hub (doctor -h for flags)\n discover Find gateways on the local network (discover -h for flags)\n traces Export, verify or summarize archived calls\n\nFlags:\n")
flag.CommandLine.VisitAll(func(f *flag.Flag) {
if f.Name == "password" {
return
Expand Down
17 changes: 17 additions & 0 deletions config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,23 @@
# The compatibility notes at the end show old spellings for existing deployments.
config-version: 8

# Durable model-call history. The current live segment stays local in plaintext (0600 on POSIX).
# See docs/trace-archive.md for encryption, recovery, and export.
# trace-archive:
# enabled: false
# dir: "" # default: <auth-dir>-traces (sibling directory)
# recipients: [] # required when enabled; use the age1... public key
# Generate on the reader machine: age-keygen -o trace-identity.txt
# Print the public recipient: age-keygen -y trace-identity.txt
# queue-size: 32
# max-local-size-mb: 0 # zero disables local eviction
# exclude-endpoints: []
# exclude-models: []
# upload:
# enabled: false
# binary: rclone
# remote: gdrive:t3code-proxy-traces

# Optional model catalog sources: http(s) URLs or absolute local file paths.
# Each explicit source overrides --local-model; otherwise --local-model uses
# embedded catalogs, and normal mode uses the official remote sources.
Expand Down
17 changes: 17 additions & 0 deletions config.hub.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,23 @@
# documented in config.example.yaml.
config-version: 8

# Durable model-call history. The current live segment stays local in plaintext (0600 on POSIX).
# See docs/trace-archive.md for encryption, recovery, and export.
# trace-archive:
# enabled: false
# dir: "" # default: <auth-dir>-traces (sibling directory)
# recipients: [] # required when enabled; use the age1... public key
# Generate on the reader machine: age-keygen -o trace-identity.txt
# Print the public recipient: age-keygen -y trace-identity.txt
# queue-size: 32
# max-local-size-mb: 0 # zero disables local eviction
# exclude-endpoints: []
# exclude-models: []
# upload:
# enabled: false
# binary: rclone
# remote: gdrive:t3code-proxy-traces

server:
# Listen on loopback only. Tailscale Serve publishes the port to your tailnet:
# tailscale serve --bg --https=8317 http://127.0.0.1:8317
Expand Down
95 changes: 95 additions & 0 deletions docs/trace-archive.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# Trace archive

Enable `trace-archive.enabled` to keep model calls for evaluation, replay or training. The default is off. The archive works independently of request logging. The web panel provides the switch, directory and recorder status under Logging. Set encryption and upload options in YAML. Settings reload with the hub configuration.

## Privacy and keys

A trace includes instructions, conversation history, tool arguments, tool results, generated content and whatever secrets a user pasted into a session. The live segment for the current UTC day stays local in plaintext (mode 0600 on POSIX) until it seals. Use a private directory on an encrypted volume, restrict its OS permissions to the hub account, and protect exports just as carefully. New directories use mode 0700 and new files use 0600 on POSIX. Set Windows ACLs on the parent directory.

Generate an age identity on the machine where you will read the archives. Keep the private identity there. Back it up somewhere other than the upload destination. Losing every matching identity makes a sealed encrypted segment unrecoverable.

```sh
age-keygen -o trace-identity.txt
age-keygen -y trace-identity.txt
```

Copy only the printed `age1...` public recipient to the hub. Any configured recipient can decrypt. A segment uses the recipients configured when it seals. A hub with public keys alone can read its live file and cannot decrypt sealed history. Enabling the archive requires at least one valid age X25519 recipient. Every newly sealed segment is encrypted.

## Configuration and Google Drive

```yaml
trace-archive:
enabled: true
dir: /srv/private/traces
recipients: [age1REPLACE_WITH_PUBLIC_RECIPIENT]
queue-size: 32
max-local-size-mb: 8192
exclude-models: []
exclude-endpoints: []
upload:
enabled: true
binary: rclone
remote: gdrive:t3code-proxy-traces
```

1. Install rclone for the OS user that runs the hub. Run `rclone config`, create a remote named `gdrive`, select Google Drive and authorize the account. Keep rclone's configuration private.
2. Run `rclone mkdir gdrive:t3code-proxy-traces` and `rclone lsf gdrive:t3code-proxy-traces` as the hub user.
3. Apply the config and make a test call. Send an authenticated POST to `/v8/management/trace-archive/seal-now`. Check `GET /v8/management/trace-archive/status` for the segment, verified destination, dropped count and errors.
4. Test recovery on the identity machine. Copy a segment and its matching manifest into one directory, then run `traces verify` and `traces export`.

The default directory is a sibling of the resolved auth directory, with `-traces` appended to its name (for example `~/.t3code-proxy-traces`). An explicit `trace-archive.dir` takes precedence. To continue using a previous archive location, set `dir` to that path. Startup renames legacy `.manifest.json` sidecars there to `.manifest`. Historical plaintext segments remain readable locally and are ineligible for upload. Filters are exact client model names and URL paths. Queue size bounds concurrent captures plus queued calls. A call exceeding the available slots, 128 MiB per call, 256 MiB of aggregate capture buffers, or 256 levels of JSON nesting is dropped whole. Disk, compression, reconstruction, encryption and networking run outside client handlers. Counters for dropped calls reset when the hub restarts. The local limit is in MiB and zero means unlimited. Retention removes the oldest sealed pairs first. With uploads enabled, only a pair verified at the configured destination qualifies. The live segment always stays on disk.

Both files are sent with `rclone copyto`. The hub streams each back with `rclone cat` to check the exact byte length and SHA-256 even on remotes that expose only MD5. This uses bandwidth in both directions. A verified pair has the destination in local and remote manifest field `uploaded_to`. Errors retry in the background with exponential backoff up to one hour. Uploads are cancellable at shutdown.

## Reading a segment without the hub

```sh
age -d -i trace-identity.txt traces-2026-11-20-123.jsonl.zst.age | zstd -dc > restored.jsonl
jq -c 'select(.type == "call") | .call.response' restored.jsonl
```

Schema version 1 uses UTF-8 JSON objects separated by LF, one object per line, including the final LF. Dates are UTC. File names are `traces-YYYY-MM-DD[-n].live` while open and `traces-YYYY-MM-DD[-n].jsonl.zst.age` once sealed. The writer uses a nanosecond suffix to distinguish same-day segments. The date is the recording day. A new day or a manual seal starts an independent blob dictionary. A current-day live file can survive a clean shutdown. Recovery truncates only the unterminated final line, rejects a corrupt complete line and seals stale live files.

Entries appear in this order:

* A header has `type: "header"`, integer `schema_version: 1` and `date`.
* A blob has `type: "blob"`, `sha256` (lowercase hex) and `value` (any JSON value). Hash the UTF-8 canonical JSON value. Objects sort keys by their UTF-8 byte strings recursively, arrays retain order, and separators have no whitespace. Strings use UTF-8 in double quotes. Escape quotes, backslashes and the five short control escapes. Encode other ASCII control characters, `<`, `>` and `&` as lowercase `\u00xx`, and U+2028/U+2029 as `\u2028`/`\u2029`. Numeric literals retain their original spelling, including large integers. Each hash appears once and before its first reference.
* A call has `type: "call"` and a `call` object with its own integer `schema_version`. Its packed `client_request` and `upstream_request` each have a `body` and optional `parts`. Copy body, then insert each parts entry at the same top-level key. A string denotes one blob hash. An array denotes an ordered array of blob hashes. Empty arrays remain empty. Fields eligible for extraction are system, instructions, tools, messages, input, contents, systemInstruction and toolConfig. The `base64` alternative holds a non-JSON request as standard padded Base64. Missing payload fields mean that side was unobserved.
* The call's `response` is the reconstructed upstream object. Claude thinking, signatures, redacted blocks, tools, citations and stop information stay in provider format. Responses items retain opaque encrypted content and summaries. Chat chunks become choices with complete messages and tool calls. For HTTP calls, a differing final client result is stored as `client_response`, or `client_raw_base64` if it cannot be reconstructed. The client-facing body supplies a fallback when upstream capture is unavailable. When reconstruction fails, `raw_frames_base64` contains ordered standard Base64 snapshots of executor chunks (scanner lines, HTTP bodies or WebSocket messages). Base64 decoding recovers each frame independently. Network headers are omitted. MCP authorization fields and recognized credentials in JSON header and metadata containers are masked before blob hashing and storage.
* Metadata includes hub version, client URL path and protocol, user agent, client name when supplied, HTTP or WS transport, stream flag, session lineage, trace and request IDs, provider, executor, non-secret auth index, requested/alias/upstream/response models, effort, tier, status, failure classification and token detail where available. Attempts are ordered by outbound send. The final upstream body is paired with the final response. Times are RFC 3339 UTC. Latency and TTFT are integer nanoseconds. `first_byte` is the first observed HTTP response byte or WS frame. `ttft_ns` uses provider token timing where available and falls back to the first observed response byte. Optional fields can be absent for early failures or providers that omit them.
* A sealed segment ends with `type: "footer"`, `counts` and `sha256`. Hash the exact decompressed bytes of every preceding line **including its LF**. Counts are calls, blobs, unique_blob_bytes, referenced_blob_bytes and jsonl_bytes (before the footer). The dedup ratio divides bytes across all blob references by unique blob bytes. A live segment has no footer.

The sidecar `<sealed filename>.manifest` contains file, date, counts, size (sealed bytes), sha256 (sealed file), source_sha256 (same digest as footer) and optional uploaded_to. The hidden writer lock and temporary files are local implementation details. Export needs only the segment pairs, matching identities for encrypted files and any live segment to include.

To re-inline both requests from a decrypted segment using jq alone:

```sh
jq -nc '
def unpack($b):
if has("body") then . as $p | .body +
(($p.parts // {}) | with_entries(.value |=
(if type == "array" then map($b[.]) else $b[.] end)))
else . end;
foreach inputs as $e ({b:{},call:null};
.call=null | if $e.type=="blob" then .b[$e.sha256]=$e.value
elif $e.type=="call" then .b as $b | .call=($e.call |
if has("client_request") then .client_request |= unpack($b) else . end |
if has("upstream_request") then .upstream_request |= unpack($b) else . end)
else . end;
.call // empty)' restored.jsonl
```

## CLI and management

```sh
t3code-proxy traces stats --dir ./traces
t3code-proxy traces verify --dir ./traces --identity trace-identity.txt
t3code-proxy traces export --dir ./traces --from 2026-11-01 --to 2026-11-20 \
--identity trace-identity.txt --output calls.jsonl
t3code-proxy traces export --dir ./traces --identity trace-identity.txt \
--format conversations > sessions.jsonl
```

Date bounds include both endpoints. Omit `--dir` to resolve it from `--config` (default `config.yaml`). Call export writes one resolved record per line. Conversation export groups records by session, putting calls without a session in individual groups. Call export holds one segment's blob dictionary in memory. Conversation export also buffers the grouped records in the selected range. Stats reads sealed manifests without a private identity. Verify checks sealed hashes, age authentication, footer and blob integrity, counts and references. Live files can only be checked through their last complete entry. Output files created with `--output` must be new paths.

Authenticated v8 routes are `GET/PUT /v8/management/trace-archive/settings`, `GET /v8/management/trace-archive/status`, and `POST /v8/management/trace-archive/seal-now`. PUT replaces the settings block. Status includes UTC-day counts and bytes, queue depth, dropped calls, dedup ratio, local manifests and writer/upload errors.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/rune-oai/t3code-proxy
go 1.26.10

require (
filippo.io/age v1.2.1
github.com/MoeForge/moego v0.1.0-alpha.3
github.com/andybalholm/brotli v1.0.6
github.com/atotto/clipboard v0.1.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o=
filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/MoeForge/moego v0.1.0-alpha.3 h1:sn8Kg4qY/gqGXU4CCJWfgzouaim6alkvXcob0uFDohg=
Expand Down
19 changes: 18 additions & 1 deletion internal/api/handlers/management/config_v8.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package management

import (
"encoding/json"
"errors"
"io"
"net/http"
"os"
Expand All @@ -17,6 +18,18 @@ import (
// ConfigV8ContextKey marks operations registered by the v8 API that save configuration.
const ConfigV8ContextKey = "management.config-v8"

// TraceArchiveSettings returns the default block even on a fresh installation.
func (h *Handler) TraceArchiveSettings(c *gin.Context) {
if c.Request.Method == http.MethodGet {
h.mu.Lock()
defer h.mu.Unlock()
c.JSON(http.StatusOK, h.cfg.TraceArchive)
return
}
c.Params = append(c.Params, gin.Param{Key: "path", Value: "/trace-archive"})
h.ConfigV8(c)
}

// ConfigV8 serves the persisted configuration using v8 names. GET is a view;
// only a successful configuration mutation migrates the stored document.
//
Expand Down Expand Up @@ -179,7 +192,11 @@ func (h *Handler) ConfigV8(c *gin.Context) {
}
next, err := config.ParseConfigBytes(data)
if err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "invalid_config", "message": err.Error()})
status := http.StatusUnprocessableEntity
if errors.Is(err, config.ErrTraceArchiveRecipient) {
status = http.StatusBadRequest
}
c.JSON(status, gin.H{"error": "invalid_config", "message": err.Error()})
return
}
if err = config.ValidateV8Config(data); err != nil {
Expand Down
6 changes: 6 additions & 0 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"sync/atomic"
"time"

"github.com/rune-oai/t3code-proxy/internal/tracearchive"

"github.com/gin-gonic/gin"
managementHandlers "github.com/rune-oai/t3code-proxy/internal/api/handlers/management"
"github.com/rune-oai/t3code-proxy/internal/api/middleware"
Expand All @@ -41,6 +43,7 @@ import (
// Server represents the main API server.
// It encapsulates the Gin engine, HTTP server, handlers, and configuration.
type Server struct {
archive tracearchive.Recorder
// engine is the Gin web framework engine instance.
engine *gin.Engine

Expand Down Expand Up @@ -228,6 +231,8 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk

engine.Use(corsMiddleware())
engine.Use(s.responseCompressionMiddleware())
s.configureArchive(cfg)
engine.Use(s.traceMiddleware)
openai.SetResponsesWebsocketCompressionEnabled(cfg.Codex.WebsocketCompressionEnabled())
s.wsAuthEnabled.Store(cfg.WebsocketAuth)
s.exampleAPIKeySafeModeActive.Store(s.exampleAPIKeySafeModeRequired(cfg))
Expand Down Expand Up @@ -453,6 +458,7 @@ func (s *Server) Start() error {
// - error: An error if the server fails to stop
func (s *Server) Stop(ctx context.Context) error {
log.Debug("Stopping API server...")
defer s.archive.Close()

if s.keepAliveEnabled {
select {
Expand Down
4 changes: 4 additions & 0 deletions internal/api/server_management_v8.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ func (s *Server) registerManagementV8Routes() {
v8.GET("/quota-history", s.mgmt.GetQuotaHistory)
v8.POST("/quota-history/refresh", s.mgmt.RefreshQuotaHistory)

v8.GET("/trace-archive/settings", s.mgmt.TraceArchiveSettings)
v8.PUT("/trace-archive/settings", s.mgmt.TraceArchiveSettings)
v8.GET("/trace-archive/status", func(c *gin.Context) { c.JSON(http.StatusOK, s.archive.Status()) })
v8.POST("/trace-archive/seal-now", s.sealTraces)
v8.GET("/observability/logs", s.mgmt.GetLogs)
v8.DELETE("/observability/logs", s.mgmt.DeleteLogs)
v8.GET("/observability/logs/errors", s.mgmt.GetRequestErrorLogs)
Expand Down
1 change: 1 addition & 0 deletions internal/api/server_reload.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ func (s *Server) UpdateClientsContext(ctx context.Context, cfg *config.Config) b
s.mgmt.SetAuthManager(s.handlers.AuthManager)
s.mgmt.SetPluginHost(s.pluginHost)
s.mgmt.ApplyQuotaHistoryConfig(cfg)
s.configureArchive(cfg)
}
s.refreshPluginManagementRoutes()

Expand Down
Loading